aggroot 1.4.9 → 1.5.1

package/scripts/test-tokenizer.js

@@ -0,0 +1,148 @@
+// Test just the tokenizer - inline the key parts
+function makeLexer(src: string) {
+  return { src, len: src.length, i: 0 }
+}
+
+function peek(L: any, off = 0): string {
+  return L.i + off < L.len ? L.src[L.i + off] : ''
+}
+
+function advance(L: any): void {
+  L.i++
+}
+
+function skipBlanks(L: any): void {
+  while (L.i < L.len) {
+    const c = L.src[L.i]
+    if (c === ' ' || c === '\t' || c === '\r') {
+      advance(L)
+    } else if (c === '\\' && (L.src[L.i + 1] === '\n' || (L.src[L.i + 1] === '\r' && L.src[L.i + 2] === '\n'))) {
+      advance(L); advance(L)
+      if (L.src[L.i - 1] === '\r') advance(L)
+    } else {
+      break
+    }
+  }
+}
+
+function isWordChar(c: string): boolean {
+  return (
+    (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+    (c >= '0' && c <= '9') || c === '_' || c === '/' || c === '.' ||
+    c === '-' || c === '+' || c === ':' || c === '@' || c === '%' ||
+    c === ',' || c === '~' || c === '^' || c === '?' || c === '*' ||
+    c === '!' || c === '=' || c === '[' || c === ']'
+  )
+}
+
+function isWordStart(c: string): boolean {
+  return isWordChar(c) || c === '\\'
+}
+
+function nextToken(L: any, ctx: string = 'arg'): any {
+  skipBlanks(L)
+  const start = L.i
+  if (L.i >= L.len) return { type: 'EOF', value: '', start, end: start }
+
+  const c = L.src[L.i]
+  const c1 = peek(L, 1)
+  const c2 = peek(L, 2)
+
+  if (c === '\n') { advance(L); return { type: 'NEWLINE', value: '\n', start, end: L.i } }
+  if (c === '#') {
+    const si = L.i
+    while (L.i < L.len && L.src[L.i] !== '\n') advance(L)
+    return { type: 'COMMENT', value: L.src.slice(si, L.i), start, end: L.i }
+  }
+
+  // Multi-char operators (longest match first)
+  if (c === '&' && c1 === '&') { advance(L); advance(L); return { type: 'OP', value: '&&', start, end: L.i } }
+  if (c === '|' && c1 === '|') { advance(L); advance(L); return { type: 'OP', value: '||', start, end: L.i } }
+  if (c === '|' && c1 === '&') { advance(L); advance(L); return { type: 'OP', value: '|&', start, end: L.i } }
+  if (c === '>' && c1 === '>') { advance(L); advance(L); return { type: 'OP', value: '>>', start, end: L.i } }
+  if (c === '>' && c1 === '&') { advance(L); advance(L); return { type: 'OP', value: '>&', start, end: L.i } }
+  if (c === '>' && c1 === '|') { advance(L); advance(L); return { type: 'OP', value: '>|', start, end: L.i } }
+  if (c === '&' && c1 === '>' && c2 === '>') { advance(L); advance(L); advance(L); return { type: 'OP', value: '&>>', start, end: L.i } }
+  if (c === '&' && c1 === '>') { advance(L); advance(L); return { type: 'OP', value: '&>', start, end: L.i } }
+  if (c === '<' && c1 === '<' && c2 === '<') { advance(L); advance(L); advance(L); return { type: 'OP', value: '<<<', start, end: L.i } }
+  if (c === '<' && c1 === '<' && c2 === '-') { advance(L); advance(L); advance(L); return { type: 'OP', value: '<<-', start, end: L.i } }
+  if (c === '<' && c1 === '<') { advance(L); advance(L); return { type: 'OP', value: '<<', start, end: L.i } }
+  if (c === '<' && c1 === '&') { advance(L); advance(L); return { type: 'OP', value: '<&', start, end: L.i } }
+  if (c === '<' && c1 === '(') { advance(L); advance(L); return { type: 'LT_PAREN', value: '<(', start, end: L.i } }
+  if (c === '>' && c1 === '(') { advance(L); advance(L); return { type: 'GT_PAREN', value: '>(', start, end: L.i } }
+  if (c === '(' && c1 === '(') { advance(L); advance(L); return { type: 'OP', value: '((', start, end: L.i } }
+  if (c === ')' && c1 === ')') { advance(L); advance(L); return { type: 'OP', value: '))', start, end: L.i } }
+
+  if (c === '|' || c === '&' || c === ';' || c === '>' || c === '<') {
+    advance(L); return { type: 'OP', value: c, start, end: L.i }
+  }
+  if (c === '(' || c === ')') {
+    advance(L); return { type: 'OP', value: c, start, end: L.i }
+  }
+
+  // In cmd position, [ [[ { start test/group
+  if (ctx === 'cmd') {
+    if (c === '[' && c1 === '[') { advance(L); advance(L); return { type: 'OP', value: '[[', start, end: L.i } }
+    if (c === '[') { advance(L); return { type: 'OP', value: '[', start, end: L.i } }
+    if (c === '{' && (c1 === ' ' || c1 === '\t' || c1 === '\n')) { advance(L); return { type: 'OP', value: '{', start, end: L.i } }
+    if (c === '}') { advance(L); return { type: 'OP', value: '}', start, end: L.i } }
+    if (c === '!' && (c1 === ' ' || c1 === '\t')) { advance(L); return { type: 'OP', value: '!', start, end: L.i } }
+  }
+
+  // Quotes
+  if (c === '"') { advance(L); return { type: 'DQUOTE', value: '"', start, end: L.i } }
+  if (c === "'") {
+    const si = L.i; advance(L)
+    while (L.i < L.len && L.src[L.i] !== "'") advance(L)
+    if (L.i < L.len) advance(L)
+    return { type: 'SQUOTE', value: L.src.slice(si, L.i), start, end: L.i }
+  }
+  if (c === '$') {
+    if (c1 === '(' && c2 === '(') { advance(L); advance(L); advance(L); return { type: 'DOLLAR_DPAREN', value: '$((', start, end: L.i } }
+    if (c1 === '(') { advance(L); advance(L); return { type: 'DOLLAR_PAREN', value: '$(', start, end: L.i } }
+    if (c1 === '{') { advance(L); advance(L); return { type: 'DOLLAR_BRACE', value: '${', start, end: L.i } }
+    advance(L); return { type: 'DOLLAR', value: '$', start, end: L.i }
+  }
+  if (c === '`') {
+    const si = L.i; advance(L)
+    while (L.i < L.len && L.src[L.i] !== '`') advance(L)
+    if (L.i < L.len) advance(L)
+    return { type: 'BACKTICK', value: L.src.slice(si, L.i), start, end: L.i }
+  }
+
+  // Word
+  if (isWordStart(c)) {
+    const si = L.i
+    while (L.i < L.len && isWordChar(L.src[L.i]!)) advance(L)
+    return { type: 'WORD', value: L.src.slice(si, L.i), start, end: L.i }
+  }
+
+  // Fallback: single char as word
+  advance(L)
+  return { type: 'WORD', value: c, start, end: L.i }
+}
+
+// Test: tokenize the whole command
+const cmd = 'cd /c/Users/zhuoz/Desktop/gui && ls -la build* 2>/dev/null || echo "No build directory found"';
+const L = makeLexer(cmd);
+const tokens: any[] = [];
+let count = 0;
+const start = Date.now();
+
+while (true) {
+  const tok = nextToken(L);
+  tokens.push(tok);
+  count++;
+  if (tok.type === 'EOF' || count > 100) break;
+  if (Date.now() - start > 5000) {
+    console.log('TOKENIZER TIMEOUT at position', L.i, 'of', L.len);
+    console.log('Last token:', JSON.stringify(tok));
+    console.log('Remaining:', JSON.stringify(cmd.slice(L.i, L.i + 50)));
+    process.exit(1);
+  }
+}
+
+console.log('Tokenization complete:', count, 'tokens in', Date.now() - start, 'ms');
+for (const t of tokens) {
+  console.log(' ', t.type, JSON.stringify(t.value));
+}