mdld-parse 0.7.2 → 0.7.4

package/src/parse.js

@@ -1,105 +1,130 @@
 import {
-    DEFAULT_CONTEXT,
     DataFactory,
     expandIRI,
-    parseSemanticBlock,
     quadIndexKey,
     createLiteral,
     hash
 } from './utils.js';
+import {
+    DEFAULT_CONTEXT,
+    URL_REGEX,
+    FENCE_REGEX,
+    PREFIX_REGEX,
+    HEADING_REGEX,
+    UNORDERED_LIST_REGEX,
+    BLOCKQUOTE_REGEX,
+    STANDALONE_SUBJECT_REGEX,
+    CARRIER_PATTERN_ARRAY,
+
+} from './constants.js';
+import {
 
-const URL_REGEX = /^[a-zA-Z][a-zA-Z0-9+.-]*:/;
-const FENCE_REGEX = /^(`{3,}|~{3,})(.*)/;
-const PREFIX_REGEX = /^\[([^\]]+)\]\s*<([^>]+)>/;
-const HEADING_REGEX = /^(#{1,6})\s+(.+?)(?:\s*(\{[^}]+\}))?$/;
-const UNORDERED_LIST_REGEX = /^(\s*)([-*+]|\d+\.)\s+(.+?)(?:\s*(\{[^}]+\}))?\s*$/;
-const BLOCKQUOTE_REGEX = /^>\s+(.+?)(?:\s*(\{[^}]+\}))?$/;
-const STANDALONE_SUBJECT_REGEX = /^\s*\{=(.*?)\}\s*$/;
-const INLINE_CARRIER_PATTERNS = {
-    EMPHASIS: /[*__`]+(.+?)[*__`]+\s*\{([^}]+)\}/y,
-    CODE_SPAN: /``(.+?)``\s*\{([^}]+)\}/y
-};
-
-// Cache for fence regex patterns to avoid recreation
-const FENCE_CLOSE_PATTERNS = new Map();
+    getFenceClosePattern,
+    calcRangeInfo,
+    calcAttrsRange,
+    createToken,
+    createCarrier,
+    createListToken,
+    parseSemCached,
+    EMPTY_SEM,
+    parseLangAndAttrs,
+    findMatchingBracket,
+    extractUrlFromBrackets,
+    extractAttributesFromText,
+    determineCarrierType,
+    calcCarrierRanges,
+    extractCleanText,
+    RDF_TYPE,
+    RDF_STATEMENT,
+    RDF_SUBJECT,
+    RDF_PREDICATE,
+    RDF_OBJECT,
+    createLeanOriginEntry,
+    resolveFragment,
+    resolveSubject,
+    resolveObject,
+    processTokenWithBlockTracking
+} from './shared.js';
 
-function getFenceClosePattern(fenceChar) {
-    if (!FENCE_CLOSE_PATTERNS.has(fenceChar)) {
-        FENCE_CLOSE_PATTERNS.set(fenceChar, new RegExp(`^(${fenceChar}{3,})`));
-    }
-    return FENCE_CLOSE_PATTERNS.get(fenceChar);
-}
 
-function parseLangAndAttrs(langAndAttrs) {
-    const spaceIndex = langAndAttrs.indexOf(' ');
-    const braceIndex = langAndAttrs.indexOf('{');
-    const langEnd = Math.min(
-        spaceIndex > -1 ? spaceIndex : Infinity,
-        braceIndex > -1 ? braceIndex : Infinity
-    );
-    return {
-        lang: langAndAttrs.substring(0, langEnd),
-        attrsText: langAndAttrs.substring(langEnd).match(/\{[^{}]*\}/)?.[0] || null
+export function parse(text, options = {}) {
+    const state = {
+        ctx: { ...DEFAULT_CONTEXT, ...(options.context || {}) },
+        df: options.dataFactory || DataFactory,
+        quads: [],
+        quadBuffer: new Map(),
+        removeSet: new Set(),
+        origin: {
+            quadIndex: new Map(),
+            blocks: new Map(),
+            documentStructure: []
+        },
+        currentSubject: null,
+        tokens: null,
+        currentTokenIndex: -1,
+        statements: [],
+        statementCandidates: new Map(),
+        currentBlock: null,
+        blockStack: []
     };
-}
 
-const semCache = {};
-const EMPTY_SEM = Object.freeze({ predicates: [], types: [], subject: null });
+    state.tokens = scanTokens(text);
 
-function parseSemCached(attrs) {
-    if (!attrs) return EMPTY_SEM;
-    let sem = semCache[attrs];
-    if (!sem) {
-        sem = Object.freeze(parseSemanticBlock(attrs));
-        semCache[attrs] = sem;
+    // Single-pass processing: resolve prefixes AND process tokens together
+    for (let i = 0; i < state.tokens.length; i++) {
+        const token = state.tokens[i];
+        state.currentTokenIndex = i;
+
+        // Handle prefix tokens immediately during main pass
+        if (token.type === 'prefix') {
+            let resolvedIri = token.iri;
+            if (token.iri.includes(':')) {
+                const colonIndex = token.iri.indexOf(':');
+                const potentialPrefix = token.iri.substring(0, colonIndex);
+                const reference = token.iri.substring(colonIndex + 1);
+                if (state.ctx[potentialPrefix] && potentialPrefix !== '@vocab') {
+                    resolvedIri = state.ctx[potentialPrefix] + reference;
+                }
+            }
+            state.ctx[token.prefix] = resolvedIri;
+            continue; // Skip token processor for prefixes
+        }
+
+        // Process all other tokens
+        TOKEN_PROCESSORS[token.type]?.(token, state);
     }
-    return sem;
-}
 
-function calcRangeInfo(line, attrs, lineStart, prefixLength, valueLength) {
-    const wsLength = prefixLength < line.length && line[prefixLength] === ' ' ? 1 :
-        line.slice(prefixLength).match(/^\s+/)?.[0]?.length || 0;
-    const valueStartInLine = prefixLength + wsLength;
-    return {
-        valueRange: [lineStart + valueStartInLine, lineStart + valueStartInLine + valueLength],
-        attrsRange: calcAttrsRange(line, attrs, lineStart)
-    };
-}
+    // Optimized quad filtering - use Set.has() instead of array.includes()
+    const quadKeys = new Set();
+    for (const quad of state.quads) {
+        quadKeys.add(quadIndexKey(quad.subject, quad.predicate, quad.object));
+    }
 
-function calcAttrsRange(line, attrs, lineStart) {
-    if (!attrs) return null;
-    const attrsStartInLine = line.lastIndexOf(attrs);
-    return attrsStartInLine >= 0 ? [lineStart + attrsStartInLine, lineStart + attrsStartInLine + attrs.length] : null;
-}
+    // Direct Set iteration - more efficient than filter()
+    const filteredRemove = [];
+    for (const quad of state.removeSet) {
+        const key = quadIndexKey(quad.subject, quad.predicate, quad.object);
+        if (!quadKeys.has(key)) {
+            filteredRemove.push(quad);
+        }
+    }
 
-function createToken(type, range, text, attrs = null, attrsRange = null, valueRange = null, extra = {}) {
-    const token = { type, range, text, attrs, attrsRange, valueRange, ...extra };
-    Object.defineProperty(token, '_carriers', {
-        enumerable: false, writable: true, value: null
-    });
-    return token;
+    return { quads: state.quads, remove: filteredRemove, statements: state.statements, origin: state.origin, context: state.ctx };
 }
 
+
+// Cache for fence regex patterns - using shared utility
+
 function getCarriers(token) {
     if (token.type === 'code') return [];
     return token._carriers || (token._carriers = extractInlineCarriers(token.text, token.range[0]));
 }
 
-const createListToken = (type, line, lineStart, pos, match) => {
-    const attrs = match[4] || null;
-    const prefix = match[1].length + (match[2] ? match[2].length : 0);
-    const rangeInfo = calcRangeInfo(line, attrs, lineStart, prefix, match[3].length);
-    return createToken(type, [lineStart, pos - 1], match[3].trim(), attrs,
-        rangeInfo.attrsRange, rangeInfo.valueRange, { indent: match[1].length });
-};
-
 function scanTokens(text) {
     const tokens = [];
     const lines = text.split('\n');
     let pos = 0;
     let codeBlock = null;
-
-    // Direct lookup instead of linear search
     const PROCESSORS = [
         { type: 'fence', test: line => FENCE_REGEX.test(line.trim()), process: handleFence },
         { type: 'content', test: () => codeBlock, process: line => codeBlock.content.push(line) },
@@ -206,16 +231,6 @@ function scanTokens(text) {
     return tokens;
 }
 
-function createCarrier(type, text, attrs, attrsRange, valueRange, range, pos, extra = {}) {
-    return { type, text, attrs, attrsRange, valueRange, range, pos, ...extra };
-}
-
-// Pre-compiled carrier patterns for better performance
-const CARRIER_PATTERN_ARRAY = [
-    ['EMPHASIS', /[*__`]+(.+?)[*__`]+\s*\{([^}]+)\}/y],
-    ['CODE_SPAN', /``(.+?)``\s*\{([^}]+)\}/y]
-];
-
 function extractInlineCarriers(text, baseOffset = 0) {
     const carriers = [];
     let pos = 0;
@@ -280,73 +295,124 @@ function extractInlineCarriers(text, baseOffset = 0) {
     return carriers;
 }
 
-function calcCarrierRanges(match, baseOffset, matchStart) {
-    const valueStart = baseOffset + matchStart + match[0].indexOf(match[1]);
-    const valueEnd = valueStart + match[1].length;
-    const attrsStart = baseOffset + matchStart + match[0].indexOf('{');
-    const attrsEnd = attrsStart + match[2].length + 2; // +2 for { and }
-    return {
-        valueRange: [valueStart, valueEnd],
-        attrsRange: [attrsStart + 1, attrsEnd - 1], // Exclude braces
-        range: [baseOffset + matchStart, attrsEnd],
-        pos: matchStart + match[0].length // pos should be relative to current text, not document
+
+function createBlockEntry(token, state) {
+    const blockId = token._blockId || hash(`${token.type}:${token.range?.[0]}:${token.range?.[1]}`);
+    token._blockId = blockId; // Store for later reference
+
+    const cleanText = extractCleanText(token);
+
+    const blockEntry = {
+        id: blockId,
+        type: token.type,
+        range: token.range,
+        text: cleanText,
+        subject: null,
+        types: [],
+        predicates: [],
+        carriers: [],
+        listLevel: token.indent || 0,
+        parentBlockId: state.blockStack.length > 0 ? state.blockStack[state.blockStack.length - 1] : null,
+        quadKeys: [] // Will be populated during quad emission
     };
+
+    // Store block and add to document structure
+    state.origin.blocks.set(blockId, blockEntry);
+    state.origin.documentStructure.push(blockEntry);
+
+    return blockEntry;
 }
 
-function findMatchingBracket(text, bracketStart) {
-    let bracketDepth = 1;
-    let bracketEnd = bracketStart + 1;
+function enrichBlockFromAnnotation(blockEntry, sem, carrier, state) {
+    // Update subject if available
+    if (sem.subject && sem.subject !== 'RESET') {
+        const resolvedSubject = resolveSubject(sem, state);
+        if (resolvedSubject) {
+            blockEntry.subject = resolvedSubject.value;
+        }
+    }
 
-    while (bracketEnd < text.length && bracketDepth > 0) {
-        if (text[bracketEnd] === '[') bracketDepth++;
-        else if (text[bracketEnd] === ']') bracketDepth--;
-        bracketEnd++;
+    // Add types
+    if (sem.types && sem.types.length > 0) {
+        sem.types.forEach(t => {
+            const typeIRI = typeof t === 'string' ? t : t.iri;
+            const expanded = expandIRI(typeIRI, state.ctx);
+            if (!blockEntry.types.includes(expanded)) {
+                blockEntry.types.push(expanded);
+            }
+        });
     }
 
-    return bracketDepth > 0 ? null : bracketEnd;
-}
+    // Add predicates
+    if (sem.predicates && sem.predicates.length > 0) {
+        sem.predicates.forEach(pred => {
+            const expandedPred = {
+                iri: expandIRI(pred.iri, state.ctx),
+                form: pred.form || '',
+                object: null // Will be filled during quad emission
+            };
+            blockEntry.predicates.push(expandedPred);
+        });
+    }
 
-function extractUrlFromBrackets(text, bracketEnd) {
-    let url = null;
-    let spanEnd = bracketEnd;
+    // Add carrier information
+    if (carrier) {
+        const carrierInfo = {
+            type: carrier.type,
+            range: carrier.range,
+            text: carrier.text,
+            subject: null,
+            predicates: []
+        };
 
-    if (text[spanEnd] === '(') {
-        const parenEnd = text.indexOf(')', spanEnd);
-        if (parenEnd !== -1) {
-            url = text.substring(spanEnd + 1, parenEnd);
-            spanEnd = parenEnd + 1;
+        // Extract carrier-specific semantics
+        if (carrier.attrs) {
+            const carrierSem = parseSemCached(carrier.attrs);
+            if (carrierSem.types) {
+                carrierInfo.predicates = carrierSem.predicates || [];
+            }
         }
-    }
 
-    return { url, spanEnd };
+        blockEntry.carriers.push(carrierInfo);
+    }
 }
 
-function extractAttributesFromText(text, spanEnd, baseOffset) {
-    let attrs = null;
-    let attrsRange = null;
-    const remaining = text.substring(spanEnd);
-
-    const wsMatch = remaining.match(/^\s+/);
-    const attrsStart = wsMatch ? wsMatch[0].length : 0;
-
-    if (remaining[attrsStart] === '{') {
-        const braceEnd = remaining.indexOf('}', attrsStart);
-        if (braceEnd !== -1) {
-            attrs = remaining.substring(attrsStart, braceEnd + 1);
-            const absStart = baseOffset + spanEnd + attrsStart;
-            attrsRange = [absStart, absStart + attrs.length];
-            spanEnd += braceEnd + 1;
-        }
+function processAnnotationWithBlockTracking(carrier, sem, state, options = {}) {
+    const { preserveGlobalSubject = false, implicitSubject = null } = options;
+
+    if (sem.subject === 'RESET') {
+        state.currentSubject = null;
+        return;
     }
 
-    return { attrs, attrsRange, finalSpanEnd: spanEnd };
-}
+    const previousSubject = state.currentSubject;
+    const newSubject = resolveSubject(sem, state);
+    const localObject = resolveObject(sem, state);
+
+    const effectiveSubject = implicitSubject || (newSubject && !preserveGlobalSubject ? newSubject : previousSubject);
+    if (newSubject && !preserveGlobalSubject && !implicitSubject) {
+        state.currentSubject = newSubject;
+    }
+    const S = preserveGlobalSubject ? (newSubject || previousSubject) : (implicitSubject || state.currentSubject);
+    if (!S) return;
+
+    const block = createBlock(
+        S.value, sem.types, sem.predicates,
+        carrier.range, carrier.attrsRange || null, carrier.valueRange || null,
+        carrier.type || null, state.ctx, carrier.text
+    );
+
+    const L = createLiteral(carrier.text, sem.datatype, sem.language, state.ctx, state.df);
+    const carrierO = carrier.url ? state.df.namedNode(expandIRI(carrier.url, state.ctx)) : null;
+    const newSubjectOrCarrierO = newSubject || carrierO;
 
-function determineCarrierType(url) {
-    if (url && !url.startsWith('=')) {
-        return { carrierType: 'link', resourceIRI: url };
+    // Enrich current block with semantic information
+    if (state.currentBlock) {
+        enrichBlockFromAnnotation(state.currentBlock, sem, carrier, state);
     }
-    return { carrierType: 'span', resourceIRI: null };
+
+    processTypeAnnotations(sem, newSubject, localObject, carrierO, S, block, state, carrier);
+    processPredicateAnnotations(sem, newSubject, previousSubject, localObject, newSubjectOrCarrierO, S, L, block, state, carrier);
 }
 
 function createBlock(subject, types, predicates, range, attrsRange, valueRange, carrierType, ctx, text) {
@@ -371,7 +437,7 @@ function createBlock(subject, types, predicates, range, attrsRange, valueRange,
     };
 }
 
-function emitQuad(quads, quadBuffer, removeSet, quadIndex, block, subject, predicate, object, dataFactory, meta = null, statements = null, statementCandidates = null) {
+function emitQuad(quads, quadBuffer, removeSet, quadIndex, block, subject, predicate, object, dataFactory, meta = null, statements = null, statementCandidates = null, state = null) {
     if (!subject || !predicate || !object) return;
 
     const quad = dataFactory.quad(subject, predicate, object);
@@ -407,29 +473,21 @@ function emitQuad(quads, quadBuffer, removeSet, quadIndex, block, subject, predi
         // Detect rdf:Statement pattern during single-pass parsing
         detectStatementPatternSinglePass(quad, dataFactory, meta, statements, statementCandidates);
 
-        // Create lean origin entry - avoid spread operator for better performance
-        const originEntry = {
-            blockId: block.id,
-            range: block.range,
-            carrierType: block.carrierType,
-            subject: subject.value,
-            predicate: predicate.value,
-            context: block.context, // Direct reference instead of spread
-            polarity: meta?.remove ? '-' : '+',
-            value: block.text || ''
-        };
+        // Create lean origin entry using shared utility
+        const originEntry = createLeanOriginEntry(block, subject, predicate, meta);
 
         quadIndex.set(quadKey, originEntry);
+
+        // Link block to this quad for reverse lookup during rendering
+        if (state.currentBlock && block.id === state.currentBlock.id) {
+            if (!state.currentBlock.quadKeys) {
+                state.currentBlock.quadKeys = [];
+            }
+            state.currentBlock.quadKeys.push(quadKey);
+        }
     }
 }
 
-// Extract RDF constants once at module level for efficiency
-const RDF_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type';
-const RDF_STATEMENT = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#Statement';
-const RDF_SUBJECT = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#subject';
-const RDF_PREDICATE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#predicate';
-const RDF_OBJECT = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#object';
-
 function detectStatementPatternSinglePass(quad, dataFactory, meta, statements = null, statementCandidates = null) {
     // Skip if not called from parse context (for testing compatibility)
     if (!statements || !statementCandidates) return;
@@ -479,30 +537,6 @@ function detectStatementPatternSinglePass(quad, dataFactory, meta, statements =
     }
 }
 
-const resolveFragment = (fragment, state) => {
-    if (!state.currentSubject) return null;
-    const subjectValue = state.currentSubject.value;
-    const hashIndex = subjectValue.indexOf('#');
-    const baseIRI = hashIndex > -1 ? subjectValue.slice(0, hashIndex) : subjectValue;
-    return state.df.namedNode(baseIRI + '#' + fragment);
-};
-
-function resolveSubject(sem, state) {
-    if (!sem.subject) return null;
-    if (sem.subject === 'RESET') {
-        state.currentSubject = null;
-        return null;
-    }
-    if (sem.subject.startsWith('=#')) return resolveFragment(sem.subject.substring(2), state);
-    return state.df.namedNode(expandIRI(sem.subject, state.ctx));
-}
-
-function resolveObject(sem, state) {
-    if (!sem.object) return null;
-    if (sem.object.startsWith('#')) return resolveFragment(sem.object.substring(1), state);
-    return state.df.namedNode(expandIRI(sem.object, state.ctx));
-}
-
 const createTypeQuad = (typeIRI, subject, state, block, entryIndex = null) => {
     const expandedType = expandIRI(typeIRI, state.ctx);
     const typeInfo = typeof entryIndex === 'object' ? entryIndex : { entryIndex, remove: false };
@@ -513,7 +547,8 @@ const createTypeQuad = (typeIRI, subject, state, block, entryIndex = null) => {
         state.df.namedNode(expandedType),
         state.df,
         { kind: 'type', token: `.${typeIRI}`, expandedType, entryIndex: typeInfo.entryIndex, remove: typeInfo.remove },
-        state.statements, state.statementCandidates
+        state.statements, state.statementCandidates,
+        state
     );
 };
 
@@ -557,43 +592,16 @@ function processPredicateAnnotations(sem, newSubject, previousSubject, localObje
             emitQuad(state.quads, state.quadBuffer, state.removeSet, state.origin.quadIndex, block,
                 role.subject, P, role.object, state.df,
                 { kind: 'pred', token: `${pred.form}${pred.iri}`, form: pred.form, expandedPredicate: P.value, entryIndex: pred.entryIndex, remove: pred.remove || false },
-                state.statements, state.statementCandidates
+                state.statements, state.statementCandidates,
+                state
             );
         }
     });
 }
 
 function processAnnotation(carrier, sem, state, options = {}) {
-    const { preserveGlobalSubject = false, implicitSubject = null } = options;
-
-    if (sem.subject === 'RESET') {
-        state.currentSubject = null;
-        return;
-    }
-
-    const previousSubject = state.currentSubject;
-    const newSubject = resolveSubject(sem, state);
-    const localObject = resolveObject(sem, state);
-
-    const effectiveSubject = implicitSubject || (newSubject && !preserveGlobalSubject ? newSubject : previousSubject);
-    if (newSubject && !preserveGlobalSubject && !implicitSubject) {
-        state.currentSubject = newSubject;
-    }
-    const S = preserveGlobalSubject ? (newSubject || previousSubject) : (implicitSubject || state.currentSubject);
-    if (!S) return;
-
-    const block = createBlock(
-        S.value, sem.types, sem.predicates,
-        carrier.range, carrier.attrsRange || null, carrier.valueRange || null,
-        carrier.type || null, state.ctx, carrier.text
-    );
-
-    const L = createLiteral(carrier.text, sem.datatype, sem.language, state.ctx, state.df);
-    const carrierO = carrier.url ? state.df.namedNode(expandIRI(carrier.url, state.ctx)) : null;
-    const newSubjectOrCarrierO = newSubject || carrierO;
-
-    processTypeAnnotations(sem, newSubject, localObject, carrierO, S, block, state, carrier);
-    processPredicateAnnotations(sem, newSubject, previousSubject, localObject, newSubjectOrCarrierO, S, L, block, state, carrier);
+    // Use the enhanced block tracking version
+    processAnnotationWithBlockTracking(carrier, sem, state, options);
 }
 
 
@@ -634,77 +642,9 @@ function processStandaloneSubject(token, state) {
 }
 
 const TOKEN_PROCESSORS = {
-    heading: (token, state) => {
-        processTokenAnnotations(token, state, token.type);
-    },
-    code: (token, state) => {
-        processTokenAnnotations(token, state, token.type);
-    },
-    blockquote: (token, state) => {
-        processTokenAnnotations(token, state, token.type);
-    },
-    para: (token, state) => {
-        processStandaloneSubject(token, state);
-        processTokenAnnotations(token, state, token.type);
-    },
-    list: (token, state) => {
-        processTokenAnnotations(token, state, token.type);
-    },
+    heading: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry),
+    code: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry),
+    blockquote: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry),
+    para: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry, [processStandaloneSubject]),
+    list: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry),
 };
-
-export function parse(text, options = {}) {
-    const state = {
-        ctx: { ...DEFAULT_CONTEXT, ...(options.context || {}) },
-        df: options.dataFactory || DataFactory,
-        quads: [],
-        quadBuffer: new Map(),
-        removeSet: new Set(),
-        origin: { quadIndex: new Map() },
-        currentSubject: null,
-        tokens: null,
-        currentTokenIndex: -1,
-        statements: [],
-        statementCandidates: new Map() // Track incomplete rdf:Statement patterns
-    };
-
-    state.tokens = scanTokens(text);
-
-    // Single loop instead of filter+forEach for better performance
-    for (const token of state.tokens) {
-        if (token.type === 'prefix') {
-            let resolvedIri = token.iri;
-            if (token.iri.includes(':')) {
-                const colonIndex = token.iri.indexOf(':');
-                const potentialPrefix = token.iri.substring(0, colonIndex);
-                const reference = token.iri.substring(colonIndex + 1);
-                if (state.ctx[potentialPrefix] && potentialPrefix !== '@vocab') {
-                    resolvedIri = state.ctx[potentialPrefix] + reference;
-                }
-            }
-            state.ctx[token.prefix] = resolvedIri;
-        }
-    }
-
-    for (let i = 0; i < state.tokens.length; i++) {
-        const token = state.tokens[i];
-        state.currentTokenIndex = i;
-        TOKEN_PROCESSORS[token.type]?.(token, state);
-    }
-
-    // Optimize array operations - avoid Array.from() and filter()
-    const quadKeys = new Set();
-    for (const quad of state.quads) {
-        quadKeys.add(quadIndexKey(quad.subject, quad.predicate, quad.object));
-    }
-
-    // Direct iteration instead of Array.from() + filter()
-    const filteredRemove = [];
-    for (const quad of state.removeSet) {
-        const key = quadIndexKey(quad.subject, quad.predicate, quad.object);
-        if (!quadKeys.has(key)) {
-            filteredRemove.push(quad);
-        }
-    }
-
-    return { quads: state.quads, remove: filteredRemove, statements: state.statements, origin: state.origin, context: state.ctx };
-}