arceus-s 1.6.4

package/dist/core/ingestion/route-extractors/response-shapes.js

@@ -0,0 +1,294 @@
+/**
+ * Response shape extraction from route handler file content.
+ * Detects .json() calls (JS/TS) and json_encode() calls (PHP),
+ * extracts top-level keys, and classifies by HTTP status code.
+ */
+/** Return the status code (group 1) from the last match, or undefined. */
+function lastMatchGroup(text, pattern) {
+    const matches = [...text.matchAll(pattern)];
+    if (matches.length === 0)
+        return undefined;
+    return parseInt(matches[matches.length - 1][1], 10);
+}
+/** Build the {responseKeys, errorKeys} result, deduplicating and omitting empty. */
+function buildShapeResult(successKeys, errKeys) {
+    return {
+        ...(successKeys.length > 0 ? { responseKeys: [...new Set(successKeys)] } : {}),
+        ...(errKeys.length > 0 ? { errorKeys: [...new Set(errKeys)] } : {}),
+    };
+}
+/**
+ * Detect an HTTP status code associated with a .json() call.
+ */
+export function detectStatusCode(content, jsonMatchPos, closingBracePos) {
+    const lookbackStart = Math.max(0, jsonMatchPos - 200);
+    const before = content.slice(lookbackStart, jsonMatchPos);
+    const statusChainMatch = before.match(/\.status\s*\(\s*(\d{3})\s*\)\s*$/);
+    if (statusChainMatch) {
+        return parseInt(statusChainMatch[1], 10);
+    }
+    if (closingBracePos > 0) {
+        const afterFirstArg = content.slice(closingBracePos + 1, closingBracePos + 150);
+        const secondArgMatch = afterFirstArg.match(/^\s*,\s*\{[^}]*status\s*:\s*(\d{3})/);
+        if (secondArgMatch) {
+            return parseInt(secondArgMatch[1], 10);
+        }
+    }
+    const extendedBefore = content.slice(Math.max(0, jsonMatchPos - 300), jsonMatchPos);
+    if (/new\s+Response\s*\(\s*JSON\s*\.stringify\s*$/.test(extendedBefore) && closingBracePos > 0) {
+        const afterStringify = content.slice(closingBracePos + 1, closingBracePos + 200);
+        const respStatusMatch = afterStringify.match(/^\s*\)\s*,\s*\{[^}]*status\s*:\s*(\d{3})/);
+        if (respStatusMatch) {
+            return parseInt(respStatusMatch[1], 10);
+        }
+    }
+    return undefined;
+}
+/**
+ * Extract response shapes from JS/TS handler file content.
+ */
+export function extractResponseShapes(content) {
+    const successKeys = [];
+    const errKeys = [];
+    const jsonPattern = /\.json\s*\(/g;
+    let jsonMatch;
+    while ((jsonMatch = jsonPattern.exec(content)) !== null) {
+        const matchPos = jsonMatch.index;
+        const startIdx = matchPos + jsonMatch[0].length;
+        let i = startIdx;
+        while (i < content.length && content[i] !== '{' && content[i] !== ')')
+            i++;
+        if (i >= content.length || content[i] !== '{')
+            continue;
+        const callKeys = [];
+        let depth = 0;
+        let keyStart = -1;
+        let inString = null;
+        let closingBracePos = -1;
+        for (let j = i; j < content.length; j++) {
+            const ch = content[j];
+            if (inString) {
+                if (ch === '\\') {
+                    j++;
+                    continue;
+                }
+                if (ch === inString)
+                    inString = null;
+                continue;
+            }
+            if (ch === '"' || ch === "'" || ch === '`') {
+                // Quoted string at depth 1 before ':' is a property key (e.g., { 'courses': data })
+                // The original parser only handled unquoted identifiers.
+                if (depth === 1 && keyStart === -1) {
+                    const quote = ch;
+                    const strStart = j + 1;
+                    let strEnd = -1;
+                    for (let s = strStart; s < content.length; s++) {
+                        if (content[s] === '\\') {
+                            s++;
+                            continue;
+                        }
+                        if (content[s] === quote) {
+                            strEnd = s;
+                            break;
+                        }
+                    }
+                    if (strEnd !== -1) {
+                        // Scan forward for ':' without allocating a substring
+                        let p = strEnd + 1;
+                        while (p < content.length &&
+                            (content[p] === ' ' ||
+                                content[p] === '\t' ||
+                                content[p] === '\n' ||
+                                content[p] === '\r'))
+                            p++;
+                        if (content[p] === ':') {
+                            callKeys.push(content.slice(strStart, strEnd));
+                        }
+                        j = strEnd;
+                        continue;
+                    }
+                }
+                inString = ch;
+                continue;
+            }
+            if (ch === '{') {
+                depth++;
+                continue;
+            }
+            if (ch === '}') {
+                depth--;
+                if (depth === 0) {
+                    closingBracePos = j;
+                    break;
+                }
+                continue;
+            }
+            if (depth !== 1)
+                continue;
+            if (keyStart === -1 && /[a-zA-Z_$]/.test(ch)) {
+                keyStart = j;
+            }
+            else if (keyStart !== -1 && !/[a-zA-Z0-9_$]/.test(ch)) {
+                const key = content.slice(keyStart, j);
+                const rest = content.slice(j).trimStart();
+                if (rest[0] === ':' || rest[0] === ',' || rest[0] === '}') {
+                    callKeys.push(key);
+                }
+                keyStart = -1;
+            }
+        }
+        if (callKeys.length === 0)
+            continue;
+        const status = detectStatusCode(content, matchPos, closingBracePos);
+        if (status !== undefined && status >= 400) {
+            errKeys.push(...callKeys);
+        }
+        else {
+            successKeys.push(...callKeys);
+        }
+    }
+    return buildShapeResult(successKeys, errKeys);
+}
+/**
+ * Find the last exit/die boundary in a string.
+ * Matches: exit; exit(N); die; die('msg'); die($var);
+ * Returns the index AFTER the boundary.
+ */
+function findLastExitBoundary(text) {
+    const pattern = /\b(exit|die)\s*(\([^)]*\))?\s*;/g;
+    let lastEnd = -1;
+    let m;
+    while ((m = pattern.exec(text)) !== null) {
+        lastEnd = m.index + m[0].length;
+    }
+    return lastEnd;
+}
+function detectPHPStatusCode(content, jsonEncodePos) {
+    const lookbackStart = Math.max(0, jsonEncodePos - 300);
+    let before = content.slice(lookbackStart, jsonEncodePos);
+    const boundaryEnd = findLastExitBoundary(before);
+    if (boundaryEnd !== -1) {
+        before = before.slice(boundaryEnd);
+    }
+    return (lastMatchGroup(before, /http_response_code\s*\(\s*(\d{3})\s*\)/g) ??
+        lastMatchGroup(before, /header\s*\(\s*['"]HTTP\/[\d.]+\s+(\d{3})/g) ??
+        // CGI/FastCGI format
+        lastMatchGroup(before, /header\s*\(\s*['"]Status:\s*(\d{3})/g));
+}
+function findMatchingBracket(content, openPos, open, close) {
+    let depth = 0;
+    let inString = null;
+    for (let j = openPos; j < content.length; j++) {
+        const ch = content[j];
+        if (inString) {
+            if (ch === '\\') {
+                j++;
+                continue;
+            }
+            if (ch === inString)
+                inString = null;
+            continue;
+        }
+        if (ch === '"' || ch === "'") {
+            inString = ch;
+            continue;
+        }
+        if (ch === open) {
+            depth++;
+            continue;
+        }
+        if (ch === close) {
+            depth--;
+            if (depth === 0)
+                return j;
+            continue;
+        }
+    }
+    return -1;
+}
+function extractPHPArrayKeys(arrayContent) {
+    const keys = [];
+    let depth = 0;
+    let inString = null;
+    const topLevelRanges = [];
+    let rangeStart = 0;
+    for (let i = 0; i < arrayContent.length; i++) {
+        const ch = arrayContent[i];
+        if (inString) {
+            if (ch === '\\') {
+                i++;
+                continue;
+            }
+            if (ch === inString)
+                inString = null;
+            continue;
+        }
+        if (ch === '"' || ch === "'") {
+            inString = ch;
+            continue;
+        }
+        if (ch === '[' || ch === '(' || ch === '{') {
+            if (depth === 0)
+                topLevelRanges.push([rangeStart, i]);
+            depth++;
+        }
+        else if (ch === ']' || ch === ')' || ch === '}') {
+            depth--;
+            if (depth === 0)
+                rangeStart = i + 1;
+        }
+    }
+    if (depth === 0)
+        topLevelRanges.push([rangeStart, arrayContent.length]);
+    for (const [start, end] of topLevelRanges) {
+        const segment = arrayContent.slice(start, end);
+        const localPattern = /(['"])([a-zA-Z_][a-zA-Z0-9_]*)\1\s*=>/g;
+        let m;
+        while ((m = localPattern.exec(segment)) !== null) {
+            keys.push(m[2]);
+        }
+    }
+    return keys;
+}
+export function extractPHPResponseShapes(content) {
+    const successKeys = [];
+    const errKeys = [];
+    const jsonEncodePattern = /json_encode\s*\(/g;
+    let match;
+    while ((match = jsonEncodePattern.exec(content)) !== null) {
+        const matchPos = match.index;
+        const startIdx = matchPos + match[0].length;
+        let i = startIdx;
+        while (i < content.length && /\s/.test(content[i]))
+            i++;
+        if (i >= content.length)
+            continue;
+        let arrayEnd = -1;
+        const openChar = content[i];
+        if (openChar === '[') {
+            arrayEnd = findMatchingBracket(content, i, '[', ']');
+        }
+        else if (content.slice(i, i + 6) === 'array(') {
+            i += 5;
+            arrayEnd = findMatchingBracket(content, i, '(', ')');
+        }
+        else {
+            continue;
+        }
+        if (arrayEnd === -1)
+            continue;
+        const arrayContent = content.slice(i + 1, arrayEnd);
+        const callKeys = extractPHPArrayKeys(arrayContent);
+        if (callKeys.length === 0)
+            continue;
+        const status = detectPHPStatusCode(content, matchPos);
+        if (status !== undefined && status >= 400) {
+            errKeys.push(...callKeys);
+        }
+        else {
+            successKeys.push(...callKeys);
+        }
+    }
+    return buildShapeResult(successKeys, errKeys);
+}

package/dist/core/ingestion/scope-extractor-bridge.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Bridge between a language provider's `emitScopeCaptures` hook and the
+ * `ScopeExtractor` (RFC #909 Ring 2 PKG #920).
+ *
+ * Extracted into its own module so it can be imported by test code
+ * without pulling in `parse-worker.ts` — which has a top-level
+ * `parentPort!.on('message', ...)` call that assumes a worker-thread
+ * context and throws on direct import.
+ *
+ * The bridge:
+ *
+ *   1. Short-circuits when the provider has NOT implemented
+ *      `emitScopeCaptures`. Returns `undefined`; zero work done. This is
+ *      the state of every language today — `ParsedFile` production stays
+ *      dormant until a language migrates.
+ *   2. Short-circuits empty / whitespace-only files. There is no scope
+ *      content to extract, and some tree-sitter queries do not match an
+ *      otherwise valid empty root node.
+ *   3. Invokes the hook + feeds its output to `ScopeExtractor.extract`.
+ *   4. **Swallows exceptions from either side.** A failure here returns
+ *      `undefined` and emits a warning via `onWarn`; legacy parsing on
+ *      the same file continues unaffected by the scope-extraction miss.
+ *      Scope-based resolution is the new path under construction — it
+ *      must not destabilize the legacy DAG.
+ */
+import type { ParsedFile } from '../../_shared/index.js';
+import type { LanguageProvider } from './language-provider.js';
+/** Callback used to report scope-extraction warnings to the host (worker or direct). */
+export type ScopeBridgeWarn = (message: string) => void;
+/**
+ * Produce a `ParsedFile` for the given file, or `undefined` when the
+ * provider hasn't migrated / the extractor throws. Never propagates
+ * exceptions.
+ */
+export declare function extractParsedFile(provider: LanguageProvider, sourceText: string, filePath: string, onWarn?: ScopeBridgeWarn, cachedTree?: unknown): ParsedFile | undefined;

package/dist/core/ingestion/scope-extractor-bridge.js

@@ -0,0 +1,49 @@
+/**
+ * Bridge between a language provider's `emitScopeCaptures` hook and the
+ * `ScopeExtractor` (RFC #909 Ring 2 PKG #920).
+ *
+ * Extracted into its own module so it can be imported by test code
+ * without pulling in `parse-worker.ts` — which has a top-level
+ * `parentPort!.on('message', ...)` call that assumes a worker-thread
+ * context and throws on direct import.
+ *
+ * The bridge:
+ *
+ *   1. Short-circuits when the provider has NOT implemented
+ *      `emitScopeCaptures`. Returns `undefined`; zero work done. This is
+ *      the state of every language today — `ParsedFile` production stays
+ *      dormant until a language migrates.
+ *   2. Short-circuits empty / whitespace-only files. There is no scope
+ *      content to extract, and some tree-sitter queries do not match an
+ *      otherwise valid empty root node.
+ *   3. Invokes the hook + feeds its output to `ScopeExtractor.extract`.
+ *   4. **Swallows exceptions from either side.** A failure here returns
+ *      `undefined` and emits a warning via `onWarn`; legacy parsing on
+ *      the same file continues unaffected by the scope-extraction miss.
+ *      Scope-based resolution is the new path under construction — it
+ *      must not destabilize the legacy DAG.
+ */
+import { extract as extractScope } from './scope-extractor.js';
+import { logger } from '../logger.js';
+/**
+ * Produce a `ParsedFile` for the given file, or `undefined` when the
+ * provider hasn't migrated / the extractor throws. Never propagates
+ * exceptions.
+ */
+export function extractParsedFile(provider, sourceText, filePath, onWarn, cachedTree) {
+    if (provider.emitScopeCaptures === undefined)
+        return undefined;
+    if (sourceText.trim().length === 0)
+        return undefined;
+    try {
+        const captures = provider.emitScopeCaptures(sourceText, filePath, cachedTree);
+        return extractScope(captures, filePath, provider);
+    }
+    catch (err) {
+        const message = `scope extraction failed for ${filePath}: ${err instanceof Error ? err.message : String(err)}`;
+        if (onWarn !== undefined)
+            onWarn(message);
+        logger.warn(message);
+        return undefined;
+    }
+}

package/dist/core/ingestion/scope-extractor.d.ts

@@ -0,0 +1,86 @@
+/**
+ * `ScopeExtractor` — the central, source-agnostic driver that turns a
+ * language provider's `CaptureMatch[]` into a `ParsedFile`
+ * (RFC §5.3 + §3.2 Phase 1; Ring 2 PKG #919).
+ *
+ * Exactly one entry point: `extract(matches, filePath, provider) → ParsedFile`.
+ * Runs a five-pass pipeline over the matches. Each pass is internal; the
+ * public contract is the output `ParsedFile`.
+ *
+ * ## Design principles
+ *
+ *   - **Source-agnostic.** Consumes `CaptureMatch[]` from providers;
+ *     doesn't know whether they came from tree-sitter queries or COBOL's
+ *     regex tagger. No `Tree` / `SyntaxNode` types leak into this file.
+ *   - **One AST walk per language.** Providers do the AST walk inside
+ *     their `emitScopeCaptures` hook; this driver does zero further
+ *     traversal — it consumes captures only.
+ *   - **Pure-ish.** The extractor itself is pure (same matches →
+ *     same ParsedFile) when providers are pure. No side effects, no I/O.
+ *   - **Centralized invariant enforcement.** Structural invariants on the
+ *     scope tree (non-module has parent; parent contains child; siblings
+ *     don't overlap) are enforced by `buildScopeTree` from Ring 2 SHARED
+ *     (#912). Malformed inputs throw `ScopeTreeInvariantError`.
+ *
+ * ## The five passes
+ *
+ *   1. **Build scope tree.** Walk `@scope.*` matches. For each, consult
+ *      `provider.resolveScopeKind` (default: suffix of the capture name).
+ *      Derive parent by lexical-range containment. Hand the resulting
+ *      `Scope[]` to `buildScopeTree` for validation.
+ *   2. **Attach declarations + local bindings.** Walk `@declaration.*`
+ *      matches. For each, build a `SymbolDefinition` and attach it to
+ *      `provider.bindingScopeFor` (default: innermost containing scope)
+ *      as `ownedDefs` + a local `BindingRef { origin: 'local' }`.
+ *   3. **Collect raw imports.** Walk `@import.*` matches. Call
+ *      `provider.interpretImport` per match; attach the returned
+ *      `ParsedImport` to the ParsedFile (not to any `Scope` — finalize
+ *      reconstructs the owning scope via `provider.importOwningScope`
+ *      during Phase 2).
+ *   4. **Collect type bindings.** Walk `@type-binding.*` matches. Call
+ *      `provider.interpretTypeBinding` per match. Attach the resulting
+ *      `TypeRef` to the innermost containing scope's `typeBindings`
+ *      (or override via `provider.bindingScopeFor` if set).
+ *   5. **Collect reference sites.** Walk `@reference.*` matches. Emit
+ *      one `ReferenceSite` per match. Classify call form via
+ *      `provider.classifyCallForm` (default: the capture's sub-tag if
+ *      present; else `'free'`).
+ *
+ * ## What gets attached where
+ *
+ *   - `Scope.bindings`     — **local bindings only** at this stage (Pass 2).
+ *                            Finalize (#915) merges imports/wildcards on top.
+ *   - `Scope.ownedDefs`    — declarations structurally owned by this scope.
+ *   - `Scope.typeBindings` — local type facts (parameter annotations, `self`).
+ *   - `Scope.imports`      — empty here. Populated by the finalize algorithm
+ *                            when it resolves `ParsedImport.targetRaw`.
+ *   - `ParsedFile.parsedImports` — every raw import in this file.
+ *   - `ParsedFile.localDefs`     — flattened union of `Scope.ownedDefs`.
+ *   - `ParsedFile.referenceSites` — pre-resolution usage facts.
+ */
+import type { CaptureMatch, ParsedFile } from '../../_shared/index.js';
+import type { LanguageProvider } from './language-provider.js';
+/**
+ * The subset of `LanguageProvider` hooks that `extract()` reads. Declared
+ * as its own type so:
+ *
+ *   - Tests can implement just these six hooks without faking the whole
+ *     `LanguageProvider` interface (which is ~40 fields including the
+ *     legacy-DAG surface).
+ *   - The extractor's dependency contract stays explicit — adding a new
+ *     hook read requires updating this type.
+ *
+ * Real callers pass a full `LanguageProvider` — structural typing makes it
+ * a `ScopeExtractorHooks` for free.
+ */
+export type ScopeExtractorHooks = Pick<LanguageProvider, 'resolveScopeKind' | 'bindingScopeFor' | 'interpretImport' | 'interpretTypeBinding' | 'classifyCallForm'>;
+/**
+ * Drive the five extraction passes and return a `ParsedFile`.
+ *
+ * Throws `ScopeTreeInvariantError` (from #912) when the provider emits
+ * captures that violate structural scope invariants. The error surfaces
+ * upward rather than being silently corrected — a malformed capture set
+ * is a bug in the provider's `emitScopeCaptures`, not a data condition
+ * to tolerate.
+ */
+export declare function extract(matches: readonly CaptureMatch[], filePath: string, provider: ScopeExtractorHooks): ParsedFile;