@sprig-and-prose/sprig-universe 0.1.0

package/src/scanner.js

@@ -0,0 +1,255 @@
+/**
+ * @fileoverview Tokenizer/scanner for Sprig universe syntax
+ */
+
+/**
+ * @typedef {Object} Token
+ * @property {string} type - Token type
+ * @property {string} value - Token value
+ * @property {SourceSpan} span - Source span
+ */
+
+/**
+ * @typedef {Object} SourceSpan
+ * @property {string} file - File path
+ * @property {{ line: number, col: number, offset: number }} start - Start position
+ * @property {{ line: number, col: number, offset: number }} end - End position
+ */
+
+const KEYWORDS = new Set([
+  'universe',
+  'anthology',
+  'series',
+  'book',
+  'chapter',
+  'concept',
+  'in',
+  'relates',
+  'and',
+  'describe',
+  'from',
+  'relationships',
+  'scene',
+  'using',
+  'actor',
+  'type',
+  'identity',
+  'file',
+  'sqlite',
+  'mysql',
+  'transforms',
+  'transform',
+  'for',
+  'list',
+  'record',
+  'repository',
+]);
+
+/**
+ * Scans input text and returns tokens with source spans
+ * @param {string} text - Input text
+ * @param {string} file - File path
+ * @returns {Token[]}
+ */
+export function scan(text, file) {
+  const tokens = [];
+  let offset = 0;
+  let line = 1;
+  let col = 1;
+
+  while (offset < text.length) {
+    const startOffset = offset;
+    const startLine = line;
+    const startCol = col;
+
+    const ch = text[offset];
+
+    // Skip whitespace (but track newlines for span calculations)
+    if (/\s/.test(ch)) {
+      if (ch === '\n') {
+        line++;
+        col = 1;
+      } else {
+        col++;
+      }
+      offset++;
+      continue;
+    }
+
+    // Comments: -- to end of line
+    if (ch === '-' && offset + 1 < text.length && text[offset + 1] === '-') {
+      while (offset < text.length && text[offset] !== '\n') {
+        offset++;
+        col++;
+      }
+      // Don't create a token for comments, just skip
+      continue;
+    }
+
+    // Single-quoted strings
+    // Only treat as string delimiter if it's clearly a string (not a contraction)
+    // A contraction is when a quote appears between two letters/digits
+    if (ch === "'") {
+      const prevCh = startOffset > 0 ? text[startOffset - 1] : null;
+      const nextCh = offset + 1 < text.length ? text[offset + 1] : null;
+      
+      // If quote is between letters/digits, it's a contraction - don't parse as string
+      // Let it be handled by identifier parsing below
+      if (prevCh && /[A-Za-z0-9]/.test(prevCh) && nextCh && /[A-Za-z0-9]/.test(nextCh)) {
+        // This is a contraction, fall through to identifier parsing
+      } else {
+        // This looks like a string delimiter
+        offset++;
+        col++;
+        let value = '';
+        let escaped = false;
+
+        while (offset < text.length) {
+          const c = text[offset];
+          if (escaped) {
+            if (c === "'" || c === '\\') {
+              value += c;
+            } else {
+              value += '\\' + c;
+            }
+            escaped = false;
+            offset++;
+            col++;
+          } else if (c === '\\') {
+            escaped = true;
+            offset++;
+            col++;
+          } else if (c === "'") {
+            offset++;
+            col++;
+            break;
+          } else {
+            value += c;
+            offset++;
+            col++;
+          }
+        }
+
+        tokens.push({
+          type: 'STRING',
+          value,
+          span: {
+            file,
+            start: { line: startLine, col: startCol, offset: startOffset },
+            end: { line, col, offset },
+          },
+        });
+        continue;
+      }
+    }
+
+    // Braces
+    if (ch === '{') {
+      tokens.push({
+        type: 'LBRACE',
+        value: '{',
+        span: {
+          file,
+          start: { line: startLine, col: startCol, offset: startOffset },
+          end: { line, col: col + 1, offset: offset + 1 },
+        },
+      });
+      offset++;
+      col++;
+      continue;
+    }
+
+    if (ch === '}') {
+      tokens.push({
+        type: 'RBRACE',
+        value: '}',
+        span: {
+          file,
+          start: { line: startLine, col: startCol, offset: startOffset },
+          end: { line, col: col + 1, offset: offset + 1 },
+        },
+      });
+      offset++;
+      col++;
+      continue;
+    }
+
+    // Dot (for namespace paths)
+    if (ch === '.') {
+      tokens.push({
+        type: 'DOT',
+        value: '.',
+        span: {
+          file,
+          start: { line: startLine, col: startCol, offset: startOffset },
+          end: { line, col: col + 1, offset: offset + 1 },
+        },
+      });
+      offset++;
+      col++;
+      continue;
+    }
+
+    // Comma
+    if (ch === ',') {
+      tokens.push({
+        type: 'COMMA',
+        value: ',',
+        span: {
+          file,
+          start: { line: startLine, col: startCol, offset: startOffset },
+          end: { line, col: col + 1, offset: offset + 1 },
+        },
+      });
+      offset++;
+      col++;
+      continue;
+    }
+
+    // Identifiers and keywords (including contractions with apostrophes)
+    if (/[A-Za-z_]/.test(ch)) {
+      let value = '';
+      while (
+        offset < text.length &&
+        (/[A-Za-z0-9_]/.test(text[offset]) || 
+         (text[offset] === "'" && offset + 1 < text.length && /[A-Za-z0-9]/.test(text[offset + 1])))
+      ) {
+        value += text[offset];
+        offset++;
+        col++;
+      }
+
+      const type = KEYWORDS.has(value) ? 'KEYWORD' : 'IDENTIFIER';
+
+      tokens.push({
+        type,
+        value,
+        span: {
+          file,
+          start: { line: startLine, col: startCol, offset: startOffset },
+          end: { line, col, offset },
+        },
+      });
+      continue;
+    }
+
+    // Unknown character - emit as error token or skip?
+    // For now, skip and continue (tolerant parsing)
+    offset++;
+    col++;
+  }
+
+  // EOF token
+  tokens.push({
+    type: 'EOF',
+    value: '',
+    span: {
+      file,
+      start: { line, col, offset },
+      end: { line, col, offset },
+    },
+  });
+
+  return tokens;
+}
+