@wrtnlabs/evidence 0.0.0

package/lib/adapters/bigquery/BigQueryFileScanner.js

@@ -0,0 +1,375 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BigQueryFileScanner = void 0;
+const BigQueryColumnType_1 = require("./BigQueryColumnType");
+const BigQueryIdentifier_1 = require("./BigQueryIdentifier");
+const BigQueryString_1 = require("./BigQueryString");
+const BigQueryWithdrawals_1 = require("./BigQueryWithdrawals");
+const SourceText_1 = require("../../internal/SourceText");
+/**
+ * Extracts supported BigQuery schema declarations from one parsed source file.
+ *
+ * The scanner publishes only explicit, complete DDL so unsupported syntax cannot
+ * reduce the inventory used to evaluate coverage.
+ */
+class BigQueryFileScanner {
+    session;
+    source;
+    /**
+     * Accumulates the serializable result for this source file.
+     *
+     * Every declaration, documentation carrier, and diagnostic returned by scan belongs here.
+     */
+    output;
+    /**
+     * Maps original source offsets without normalizing line endings.
+     *
+     * Comment attachment relies on the physical UTF-16 positions from this source snapshot.
+     */
+    text;
+    /**
+     * Binds one completed GoogleSQL tree to the source it represents.
+     *
+     * The scanner borrows both inputs for this extraction and never retains parser nodes in its result.
+     */
+    constructor(session, source) {
+        this.session = session;
+        this.source = source;
+        this.output = {
+            source,
+            declarations: [],
+            documentation: [],
+            diagnostics: [],
+            complete: true,
+        };
+        this.text = new SourceText_1.SourceText(source.content);
+    }
+    /**
+     * Extracts declarations and documentation from the supported BigQuery DDL surface.
+     *
+     * Unsupported statements mark the analysis incomplete before the result is returned.
+     */
+    scan() {
+        for (const node of this.session.root.namedChildren) {
+            if (node.type === "comment")
+                continue;
+            if (node.type === "create_table_statement")
+                this.table(node);
+            else
+                this.fail(node, `The ${node.type} statement is outside the explicit CREATE TABLE schema surface.`);
+        }
+        const comments = this.session
+            .captures("(comment) @comment")
+            .map((capture) => capture.node);
+        for (let index = 0; index < comments.length; ++index) {
+            const first = comments[index];
+            if (first === undefined)
+                continue;
+            let last = first;
+            const prefix = first.text.startsWith("--")
+                ? "--"
+                : first.text.startsWith("#")
+                    ? "#"
+                    : undefined;
+            while (prefix !== undefined && this.standalone(first.startIndex)) {
+                const next = comments[index + 1];
+                if (next === undefined ||
+                    !this.standalone(next.startIndex) ||
+                    !next.text.startsWith(prefix) ||
+                    !/^[ \t\r]*\n[ \t]*$/u.test(this.source.content.slice(last.endIndex, next.startIndex)))
+                    break;
+                last = next;
+                ++index;
+            }
+            this.comment(first, last);
+        }
+        for (const documentation of this.output.documentation) {
+            const sites = new Set(documentation.attachments.map((attachment) => attachment.siteId));
+            documentation.attachments = this.output.declarations
+                .filter((declaration) => sites.has(declaration.site.id))
+                .map((declaration) => ({
+                declarationId: declaration.id,
+                siteId: declaration.site.id,
+            }));
+        }
+        BigQueryWithdrawals_1.BigQueryWithdrawals.apply(this.output);
+        return this.output;
+    }
+    /**
+     * Publishes a table only when its full schema is declared in this statement.
+     *
+     * Query-derived and inferred schemas cannot define a reliable Evidence inventory.
+     */
+    table(node) {
+        const name = node.childForFieldName("table_name");
+        const parameters = node.namedChildren.find((child) => child.type === "create_table_parameters");
+        if (name === null ||
+            parameters === undefined ||
+            node.namedChildren.some((child) => child.type === "query_statement") ||
+            !/^CREATE\s+(?:OR\s+REPLACE\s+)?(?:TEMP(?:ORARY)?\s+)?TABLE\b/iu.test(node.text)) {
+            this.fail(node, "Query-derived tables and views need a fully declared CREATE TABLE schema without AS, LIKE, COPY, or CLONE.");
+            return;
+        }
+        const path = BigQueryIdentifier_1.BigQueryIdentifier.table(name.text);
+        if (path === undefined) {
+            this.fail(name, "The table identifier cannot be represented as an explicit project.dataset.table path.");
+            return;
+        }
+        if (node.namedChildren.some((child) => child.type === "keyword_replace" ||
+            child.type === "keyword_if_not_exists")) {
+            this.fail(node, "Conditional or replacing CREATE TABLE depends on prior database state.");
+            return;
+        }
+        const model = this.declare(node, path, "model", undefined, !node.namedChildren.some((child) => child.type === "keyword_temporary"));
+        this.options(node, model);
+        const columns = parameters.namedChildren.filter((child) => child.type === "column_definition");
+        const primaryKeys = this.session.captures("(primary_key) @primary", parameters).length +
+            columns.filter((column) => column.childForFieldName("constraint_clause")?.type === "PRIMARY_KEY").length;
+        if (primaryKeys > 1)
+            this.fail(parameters, "A BigQuery table can declare only one primary key.");
+        for (const column of columns)
+            this.column(column, model, path);
+        for (const constraint of parameters.namedChildren.filter((child) => child.type === "constraint_definition"))
+            this.constraint(constraint, model, columns);
+    }
+    /**
+     * Publishes a column or nested STRUCT field under its table owner.
+     *
+     * Field paths preserve nested schema identity while ownership remains with the containing table.
+     */
+    column(node, model, ownerPath) {
+        const identifier = node.childForFieldName("column_name");
+        const name = identifier === null
+            ? undefined
+            : BigQueryIdentifier_1.BigQueryIdentifier.column(identifier.text);
+        const type = node.childForFieldName("column_type");
+        if (name === undefined || type === null) {
+            this.fail(node, "The schema field needs an explicit supported name and type.");
+            return;
+        }
+        const path = [...ownerPath, name];
+        if (!BigQueryColumnType_1.BigQueryColumnType.supported(type))
+            this.fail(type, "This field type is outside the declared GoogleSQL scalar, STRUCT, and ARRAY type surface.");
+        const column = this.declare(node, path, "column", model, model.public);
+        if (ownerPath.length !== model.address.length &&
+            node.childForFieldName("constraint_clause") !== null)
+            this.fail(node, "Primary and foreign key constraints cannot be declared on STRUCT or ARRAY elements.");
+        this.options(node, column);
+        for (const nested of type.namedChildren.filter((child) => child.type === "column_definition"))
+            this.column(nested, model, path);
+        const reference = node.namedChildren.find((child) => child.type === "foreign_key_references");
+        if (reference !== undefined) {
+            if (ownerPath.length !== model.address.length)
+                this.fail(node, "Foreign keys must belong to top-level table fields.");
+            else
+                this.relation(node, reference, model, [name], undefined);
+        }
+        this.enforcement(node);
+    }
+    /**
+     * Interprets constraints that affect the declared table surface.
+     *
+     * Primary keys remain table semantics, while foreign keys become independently selectable relations.
+     */
+    constraint(node, model, columns) {
+        this.enforcement(node);
+        const key = node.namedChildren.find((child) => child.type === "foreign_key" || child.type === "primary_key");
+        if (key === undefined) {
+            this.fail(node, "This constraint has no supported declared primary or foreign key.");
+            return;
+        }
+        const list = key.childForFieldName("column_list");
+        const names = list === null
+            ? []
+            : list.namedChildren.map((child) => BigQueryIdentifier_1.BigQueryIdentifier.canonicalColumn(child.text));
+        const available = columns
+            .map((column) => column.childForFieldName("column_name"))
+            .map((name) => name === null
+            ? undefined
+            : BigQueryIdentifier_1.BigQueryIdentifier.canonicalColumn(name.text));
+        if (names.length === 0 ||
+            names.some((name) => name === undefined || !available.includes(name)) ||
+            new Set(names).size !== names.length) {
+            this.fail(node, "Key columns must identify distinct explicitly declared top-level fields in the owning table.");
+            return;
+        }
+        if (key.type === "primary_key")
+            return;
+        const reference = key.namedChildren.find((child) => child.type === "foreign_key_references");
+        if (reference === undefined) {
+            this.fail(node, "The foreign key has no explicit referenced table and columns.");
+            return;
+        }
+        const identifier = node.childForFieldName("constraint_name");
+        const constraintName = identifier === null
+            ? undefined
+            : BigQueryIdentifier_1.BigQueryIdentifier.member(identifier.text);
+        if (identifier !== null && constraintName === undefined) {
+            this.fail(identifier, "The named foreign key uses an unsupported identifier escape or qualification.");
+            return;
+        }
+        this.relation(node, reference, model, names.filter((name) => name !== undefined), constraintName);
+    }
+    /**
+     * Publishes a foreign-key relation with a stable endpoint-based identity.
+     *
+     * Anonymous composite keys must not depend on their incidental statement order.
+     */
+    relation(node, reference, model, columns, name) {
+        const target = reference.childForFieldName("referenced_table_name");
+        const path = target === null ? undefined : BigQueryIdentifier_1.BigQueryIdentifier.table(target.text);
+        const list = reference.childForFieldName("referenced_column_list");
+        const endpoints = list === null
+            ? []
+            : list.namedChildren.map((child) => BigQueryIdentifier_1.BigQueryIdentifier.canonicalColumn(child.text));
+        if (path === undefined ||
+            endpoints.length !== columns.length ||
+            endpoints.some((part) => part === undefined) ||
+            new Set(endpoints).size !== endpoints.length) {
+            this.fail(node, "Foreign keys require explicit table paths and equally sized, distinct column endpoints.");
+            return;
+        }
+        const relationName = name ??
+            `foreign key ${JSON.stringify([columns.map((column) => column.toLowerCase()), path, endpoints])}`;
+        const declaration = this.declare(node, [...model.address, relationName], "relation", model, model.public);
+        declaration.identity = [
+            ...model.identity,
+            name === undefined ? relationName : name.toLowerCase(),
+        ];
+    }
+    /**
+     * Rejects key constraints that claim unsupported enforcement semantics.
+     *
+     * BigQuery keys still describe schema, but enforcement changes their operational meaning.
+     */
+    enforcement(node) {
+        for (const capture of this.session.captures("(constraint_enfoce_option) @enforcement", node))
+            if (!/^NOT\s+ENFORCED$/iu.test(capture.node.text))
+                this.fail(capture.node, "GoogleSQL key declarations must say NOT ENFORCED.");
+    }
+    /**
+     * Reads description strings from OPTIONS clauses attached to one declaration.
+     *
+     * Restricting the search to this node prevents nested or unrelated strings becoming documentation.
+     */
+    options(node, declaration) {
+        const clause = node.namedChildren.find((child) => child.type === "option_clause");
+        if (clause === undefined)
+            return;
+        const descriptions = clause.namedChildren.filter((child) => child.type === "option_item" &&
+            BigQueryIdentifier_1.BigQueryIdentifier.canonical(child.childForFieldName("key")?.text ?? "") === "description");
+        if (descriptions.length > 1)
+            this.fail(clause, "Each declaration can have only one OPTIONS description.");
+        for (const item of descriptions) {
+            const value = item.childForFieldName("value");
+            const mapped = value?.type === "string"
+                ? BigQueryString_1.BigQueryString.read(value.text, value.startIndex)
+                : undefined;
+            if (value === null || value === undefined || mapped === undefined) {
+                this.fail(item, "OPTIONS description requires one supported static GoogleSQL string literal.");
+                continue;
+            }
+            this.output.documentation.push({
+                id: `${declaration.id}:description`,
+                range: this.session.range(value),
+                mapped,
+                attachments: [
+                    { declarationId: declaration.id, siteId: declaration.site.id },
+                ],
+            });
+        }
+    }
+    /**
+     * Attaches contiguous standalone comments to their following declarations.
+     *
+     * Comments embedded in examples or strings stay inert because they are not documentation carriers.
+     */
+    comment(node, last) {
+        const following = this.output.declarations
+            .filter((declaration) => declaration.site.range.start.offset >= last.endIndex)
+            .sort((left, right) => left.site.range.start.offset - right.site.range.start.offset)[0];
+        const gap = following === undefined
+            ? ""
+            : this.source.content.slice(last.endIndex, following.site.range.start.offset);
+        const attached = this.standalone(node.startIndex) &&
+            following !== undefined &&
+            /^\s*$/u.test(gap) &&
+            !/\n\s*\n/u.test(gap);
+        const opening = node.text.startsWith("/*")
+            ? "/*"
+            : node.text.startsWith("--")
+                ? "--"
+                : "#";
+        this.output.documentation.push({
+            id: `${this.source.id}:comment:${node.startIndex}`,
+            range: this.text.range(node.startIndex, last.endIndex),
+            syntax: {
+                opening,
+                closing: opening === "/*" ? "*/" : "",
+                linePrefix: opening === "/*" ? "*" : opening,
+                tagBoundaries: true,
+                allowWithdrawal: attached,
+            },
+            attachments: attached
+                ? [{ declarationId: following.id, siteId: following.site.id }]
+                : [],
+        });
+    }
+    /**
+     * Determines whether a comment begins on otherwise blank source text.
+     *
+     * Trailing comments must not be reassigned as documentation for the next declaration.
+     */
+    standalone(start) {
+        const line = this.source.content.lastIndexOf("\n", start - 1) + 1;
+        return this.source.content.slice(line, start).trim() === "";
+    }
+    /**
+     * Constructs one serializable declaration site with exact source ranges.
+     *
+     * The site records physical location separately from the semantic identity used across files.
+     */
+    declare(node, path, symbol, owner, visible) {
+        const id = `${this.source.id}:${symbol}:${node.startIndex}:${JSON.stringify(path)}`;
+        const range = this.text.range(node.startIndex, node.endIndex);
+        const identity = owner === undefined
+            ? path
+            : path.map((part, index) => index < owner.address.length ? part : part.toLowerCase());
+        const declaration = {
+            id,
+            name: path.at(-1) ?? "",
+            symbol,
+            identity,
+            address: path,
+            site: {
+                id: `${this.source.id}:site:${node.startIndex}:${node.endIndex}`,
+                file: this.source.physicalPath,
+                range,
+                content: [range],
+            },
+            ...(owner === undefined ? {} : { ownerDeclarationId: owner.id }),
+            public: visible,
+        };
+        this.output.declarations.push(declaration);
+        return declaration;
+    }
+    /**
+     * Marks unsupported schema-changing syntax as an analysis failure.
+     *
+     * This preserves the failed scan instead of allowing a smaller inventory to pass coverage.
+     */
+    fail(node, message) {
+        this.output.complete = false;
+        this.output.diagnostics.push({
+            code: "bigquery-unsupported-syntax",
+            severity: "error",
+            message,
+            repair: "Select explicit GoogleSQL CREATE TABLE declarations with supported fields, descriptions, and NOT ENFORCED keys, or add extraction support for this construct.",
+            location: {
+                file: this.source.physicalPath,
+                range: this.session.range(node),
+            },
+        });
+    }
+}
+exports.BigQueryFileScanner = BigQueryFileScanner;

package/lib/adapters/bigquery/BigQueryIdentifier.d.ts

@@ -0,0 +1,37 @@
+/** Decodes GoogleSQL names without inferring an ambient project or dataset.
+ *
+ * Explicit segments preserve schema identity across snapshots, avoiding an
+ * environment-dependent address when source omits a project or dataset.
+ */
+export declare namespace BigQueryIdentifier {
+    /** Splits a declared table path into explicit identity segments.
+     *
+     * A whole-path backtick pair still permits segment boundaries at dots, matching
+     * BigQuery table qualification rather than treating it as one member name.
+     */
+    function table(raw: string): string[] | undefined;
+    /** Decodes one member identifier without splitting its literal content.
+     *
+     * Unsupported escapes and line breaks are rejected because they cannot form a
+     * stable selector segment.
+     */
+    function member(raw: string): string | undefined;
+    /** Normalizes a case-insensitive field or constraint identifier.
+     *
+     * Table paths retain their declared spelling; only member comparisons use this
+     * canonical form.
+     */
+    function canonical(raw: string): string | undefined;
+    /**
+     * Accepts documented flexible column characters, including quoted whitespace and Unicode letters.
+     *
+     * Field validation remains separate from table-path validation because their grammars differ.
+     */
+    function column(raw: string): string | undefined;
+    /**
+     * Normalizes valid field endpoints independently of case-sensitive table paths.
+     *
+     * Foreign-key endpoint comparison needs one stable spelling for supported field names.
+     */
+    function canonicalColumn(raw: string): string | undefined;
+}

package/lib/adapters/bigquery/BigQueryIdentifier.js

@@ -0,0 +1,88 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BigQueryIdentifier = void 0;
+/** Decodes GoogleSQL names without inferring an ambient project or dataset.
+ *
+ * Explicit segments preserve schema identity across snapshots, avoiding an
+ * environment-dependent address when source omits a project or dataset.
+ */
+var BigQueryIdentifier;
+(function (BigQueryIdentifier) {
+    /** Splits a declared table path into explicit identity segments.
+     *
+     * A whole-path backtick pair still permits segment boundaries at dots, matching
+     * BigQuery table qualification rather than treating it as one member name.
+     */
+    function table(raw) {
+        const quoted = raw.startsWith("`") &&
+            raw.endsWith("`") &&
+            !raw.slice(1, -1).includes("`");
+        const value = quoted ? raw.slice(1, -1) : raw;
+        const parts = value.split(".").map((part, index) => {
+            if (quoted)
+                return member(`\`${part}\``);
+            if (index === 0 && /^[A-Za-z_][A-Za-z0-9_-]*$/u.test(part))
+                return part;
+            return member(part);
+        });
+        if (parts.length > 3 || parts.some((part) => part === undefined))
+            return undefined;
+        return parts.filter((part) => part !== undefined);
+    }
+    BigQueryIdentifier.table = table;
+    /** Decodes one member identifier without splitting its literal content.
+     *
+     * Unsupported escapes and line breaks are rejected because they cannot form a
+     * stable selector segment.
+     */
+    function member(raw) {
+        if (raw.startsWith("`") && raw.endsWith("`")) {
+            const value = raw.slice(1, -1);
+            return value.length > 0 && !/[`\\\r\n]/u.test(value) ? value : undefined;
+        }
+        return /^[A-Za-z_][A-Za-z0-9_]*$/u.test(raw) ? raw : undefined;
+    }
+    BigQueryIdentifier.member = member;
+    /** Normalizes a case-insensitive field or constraint identifier.
+     *
+     * Table paths retain their declared spelling; only member comparisons use this
+     * canonical form.
+     */
+    function canonical(raw) {
+        const value = member(raw);
+        return value === undefined ? undefined : value.toLowerCase();
+    }
+    BigQueryIdentifier.canonical = canonical;
+    /**
+     * Accepts documented flexible column characters, including quoted whitespace and Unicode letters.
+     *
+     * Field validation remains separate from table-path validation because their grammars differ.
+     */
+    function column(raw) {
+        const value = member(raw);
+        if (value === undefined)
+            return undefined;
+        const characters = Array.from(value);
+        return characters.length <= 300 &&
+            characters.every((character) => /^\p{L}$/u.test(character) ||
+                /^\p{N}$/u.test(character) ||
+                /^\p{Pc}$/u.test(character) ||
+                /^\p{Pd}$/u.test(character) ||
+                /^\p{M}$/u.test(character) ||
+                /^[&%=+:'<>#|]$/u.test(character) ||
+                /^\s$/u.test(character))
+            ? value
+            : undefined;
+    }
+    BigQueryIdentifier.column = column;
+    /**
+     * Normalizes valid field endpoints independently of case-sensitive table paths.
+     *
+     * Foreign-key endpoint comparison needs one stable spelling for supported field names.
+     */
+    function canonicalColumn(raw) {
+        const value = column(raw);
+        return value === undefined ? undefined : value.toLowerCase();
+    }
+    BigQueryIdentifier.canonicalColumn = canonicalColumn;
+})(BigQueryIdentifier || (exports.BigQueryIdentifier = BigQueryIdentifier = {}));

package/lib/adapters/bigquery/BigQueryString.d.ts

@@ -0,0 +1,14 @@
+import type { IEvidenceDocumentation } from "../../structures/IEvidenceDocumentation";
+/**
+ * Decodes static GoogleSQL descriptions while retaining original UTF-16 coordinates.
+ *
+ * The BigQuery scanner uses these mappings when an OPTIONS description hosts annotations.
+ */
+export declare namespace BigQueryString {
+    /**
+     * Returns no mapping for unsupported byte strings, escapes, or concatenated expressions.
+     *
+     * A missing result makes the enclosing declaration incomplete instead of guessing offsets.
+     */
+    function read(raw: string, start: number): IEvidenceDocumentation | undefined;
+}

package/lib/adapters/bigquery/BigQueryString.js

@@ -0,0 +1,82 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BigQueryString = void 0;
+/**
+ * Decodes static GoogleSQL descriptions while retaining original UTF-16 coordinates.
+ *
+ * The BigQuery scanner uses these mappings when an OPTIONS description hosts annotations.
+ */
+var BigQueryString;
+(function (BigQueryString) {
+    /**
+     * Returns no mapping for unsupported byte strings, escapes, or concatenated expressions.
+     *
+     * A missing result makes the enclosing declaration incomplete instead of guessing offsets.
+     */
+    function read(raw, start) {
+        const head = /^(r?)("""|'''|"|')/iu.exec(raw);
+        if (head === null)
+            return undefined;
+        const delimiter = head[2];
+        if (delimiter === undefined || !raw.endsWith(delimiter))
+            return undefined;
+        const rawMode = (head[1] ?? "").toLowerCase() === "r";
+        const end = raw.length - delimiter.length;
+        let text = "";
+        const offsets = [];
+        const ends = [];
+        for (let cursor = head[0].length; cursor < end;) {
+            const from = cursor;
+            let value = raw[cursor++] ?? "";
+            if (value === "\\" && !rawMode) {
+                const escape = raw[cursor++];
+                const simple = escape === undefined ? undefined : ESCAPES.get(escape);
+                if (simple !== undefined)
+                    value = simple;
+                else if (escape === "u" || escape === "U" || escape === "x") {
+                    const width = escape === "U" ? 8 : escape === "u" ? 4 : 2;
+                    const digits = raw.slice(cursor, cursor + width);
+                    if (digits.length !== width || !/^[\da-f]+$/iu.test(digits))
+                        return undefined;
+                    const point = Number.parseInt(digits, 16);
+                    if (point > 0x10ffff || (point >= 0xd800 && point <= 0xdfff))
+                        return undefined;
+                    value = String.fromCodePoint(point);
+                    cursor += width;
+                }
+                else
+                    return undefined;
+            }
+            if (cursor > end)
+                return undefined;
+            text += value;
+            for (let index = 0; index < value.length; ++index) {
+                offsets.push(start + from);
+                ends.push(start + cursor);
+            }
+        }
+        offsets.push(start + end);
+        return {
+            hostId: "",
+            text,
+            offsets,
+            ends,
+            tagBoundaries: true,
+            allowWithdrawal: true,
+        };
+    }
+    BigQueryString.read = read;
+    const ESCAPES = new Map([
+        ["n", "\n"],
+        ["r", "\r"],
+        ["t", "\t"],
+        ["b", "\b"],
+        ["f", "\f"],
+        ["v", "\v"],
+        ["a", "\x07"],
+        ["\\", "\\"],
+        ["'", "'"],
+        ['"', '"'],
+        ["?", "?"],
+    ]);
+})(BigQueryString || (exports.BigQueryString = BigQueryString = {}));

package/lib/adapters/bigquery/BigQueryWithdrawals.d.ts

@@ -0,0 +1,14 @@
+import type { ISqlFileAnalysis } from "../sql/ISqlFileAnalysis";
+/**
+ * Propagates STRUCT withdrawals while keeping each field owned by its table model.
+ *
+ * Nested fields inherit only parsed withdrawal annotations from their enclosing schema field.
+ */
+export declare namespace BigQueryWithdrawals {
+    /**
+     * Copies parsed withdrawal annotations to explicitly nested schema fields.
+     *
+     * This runs after documentation attachment so ordinary annotations remain at their own sites.
+     */
+    function apply(analysis: ISqlFileAnalysis): void;
+}