@nshiab/simple-data-analysis-core 2.1.0 → 2.1.2

package/script/helpers/normalizeCacheCode.js

@@ -0,0 +1,334 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = normalizeCacheCode;
+const lineTerminator = /[\r\n\u2028\u2029]/u;
+const word = /^[$_\p{ID_Start}][$\u200c\u200d\p{ID_Continue}]*/u;
+const number = /^(?:0[xX][\da-fA-F_]+n?|0[bB][01_]+n?|0[oO][0-7_]+n?|(?:\d[\d_]*(?:\.[\d_]*)?|\.[\d_]+)(?:[eE][+-]?[\d_]+)?n?)/;
+const punctuation = /^(?:>>>=|===|!==|\*\*=|&&=|\|\|=|\?\?=|>>>|<<=|>>=|=>|==|!=|<=|>=|\+\+|--|&&|\|\||\?\?|\?\.(?!\d)|\*\*|<<|>>|\+=|-=|\*=|%=|&=|\|=|\^=|\.\.\.|[{}()[\].,;:?~!+*%<>=&|^\-#])/;
+const expressionPrefixes = new Set([
+    "(",
+    "[",
+    "{",
+    ",",
+    ";",
+    ":",
+    "?",
+    "=",
+    "=>",
+    "!",
+    "~",
+    "+",
+    "-",
+    "*",
+    "**",
+    "%",
+    "&",
+    "|",
+    "^",
+    "&&",
+    "||",
+    "??",
+    "==",
+    "===",
+    "!=",
+    "!==",
+    "<",
+    ">",
+    "<=",
+    ">=",
+    "<<",
+    ">>",
+    ">>>",
+    "+=",
+    "-=",
+    "*=",
+    "**=",
+    "%=",
+    "&=",
+    "|=",
+    "^=",
+    "&&=",
+    "||=",
+    "??=",
+    "<<=",
+    ">>=",
+    ">>>=",
+    "/",
+    "/=",
+]);
+const expressionKeywords = new Set([
+    "return",
+    "throw",
+    "case",
+    "delete",
+    "void",
+    "typeof",
+    "new",
+    "in",
+    "instanceof",
+    "yield",
+    "await",
+    "else",
+    "do",
+]);
+const restrictedLineBreaks = new Set([
+    "return",
+    "throw",
+    "break",
+    "continue",
+    "yield",
+    "async",
+    "++",
+    "--",
+]);
+/**
+ * Normalizes comments, whitespace, and known optional punctuation without
+ * dependencies or executing user code. Literal contents and token boundaries
+ * stay intact. Potentially significant line breaks are retained; ambiguous
+ * slash syntax falls back to exact source instead of risking a stale cache hit.
+ */
+function normalizeCacheCode(source) {
+    let position = 0;
+    let uncertain = false;
+    function quoted(quote) {
+        const start = position++;
+        while (position < source.length) {
+            const character = source[position++];
+            if (character === "\\") {
+                position++;
+            }
+            else if (character === quote) {
+                return source.slice(start, position);
+            }
+            else if (lineTerminator.test(character)) {
+                return undefined;
+            }
+        }
+    }
+    function template() {
+        const parts = [];
+        let start = ++position;
+        while (position < source.length) {
+            if (source[position] === "\\") {
+                position += 2;
+            }
+            else if (source[position] === "`") {
+                parts.push(source.slice(start, position++));
+                return JSON.stringify(parts);
+            }
+            else if (source.startsWith("${", position)) {
+                parts.push(source.slice(start, position));
+                position += 2;
+                const tokens = scan(true);
+                if (uncertain)
+                    return undefined;
+                parts.push(serialize(tokens));
+                start = position;
+            }
+            else {
+                position++;
+            }
+        }
+    }
+    function regex() {
+        const start = position++;
+        let inClass = false;
+        while (position < source.length) {
+            const character = source[position++];
+            if (lineTerminator.test(character))
+                return undefined;
+            if (character === "\\") {
+                position++;
+            }
+            else if (character === "[") {
+                // Nested character classes depend on the v flag. Leave these opaque.
+                if (inClass)
+                    return undefined;
+                inClass = true;
+            }
+            else if (character === "]") {
+                inClass = false;
+            }
+            else if (character === "/" && !inClass) {
+                const flags = source.slice(position).match(word)?.[0] ?? "";
+                position += flags.length;
+                return source.slice(start, position);
+            }
+        }
+    }
+    function scan(interpolation = false) {
+        const tokens = [];
+        let braces = 0;
+        let lineBreak = false;
+        while (position < source.length) {
+            const rest = source.slice(position);
+            const character = source[position];
+            if (/\s/u.test(character)) {
+                lineBreak ||= lineTerminator.test(character);
+                position++;
+                continue;
+            }
+            if (rest.startsWith("//")) {
+                while (position < source.length && !lineTerminator.test(source[position])) {
+                    position++;
+                }
+                continue;
+            }
+            if (rest.startsWith("/*")) {
+                const end = source.indexOf("*/", position + 2);
+                if (end === -1)
+                    break;
+                lineBreak ||= lineTerminator.test(source.slice(position, end + 2));
+                position = end + 2;
+                continue;
+            }
+            if (interpolation && character === "}" && braces === 0) {
+                position++;
+                return tokens;
+            }
+            // Legacy HTML comments and native function text are not ordinary tokens.
+            if (rest.startsWith("<!--") || rest.startsWith("-->") ||
+                rest.startsWith("[native code]"))
+                break;
+            const previous = tokens.at(-1);
+            let kind = "punctuation";
+            let text;
+            if (character === "'" || character === '"') {
+                kind = "literal";
+                text = quoted(character);
+            }
+            else if (character === "`") {
+                kind = "template";
+                text = template();
+            }
+            else if (character === "/") {
+                const property = [".", "?."].includes(tokens.at(-2)?.text ?? "");
+                if (!previous ||
+                    (previous.kind === "punctuation" &&
+                        expressionPrefixes.has(previous.text)) ||
+                    (previous.kind === "word" && !property &&
+                        expressionKeywords.has(previous.text))) {
+                    kind = "literal";
+                    text = regex();
+                }
+                else if (previous.kind === "literal" || previous.kind === "template") {
+                    text = rest.startsWith("/=") ? "/=" : "/";
+                    position += text.length;
+                }
+                else {
+                    // After a closing block/parenthesis or contextual keyword, deciding
+                    // between division and a regex can require a full JavaScript parser.
+                    break;
+                }
+            }
+            else {
+                text = rest.match(word)?.[0];
+                if (text !== undefined) {
+                    kind = "word";
+                }
+                else {
+                    text = rest.match(number)?.[0];
+                    if (text !== undefined)
+                        kind = "literal";
+                    else
+                        text = rest.match(punctuation)?.[0];
+                }
+                if (text !== undefined)
+                    position += text.length;
+            }
+            if (text === undefined) {
+                uncertain = true;
+                break;
+            }
+            if (kind === "punctuation") {
+                if (text === "{")
+                    braces++;
+                if (text === "}")
+                    braces--;
+            }
+            tokens.push({ kind, text, lineBreak });
+            lineBreak = false;
+        }
+        if (position < source.length || interpolation)
+            uncertain = true;
+        return tokens;
+    }
+    const tokens = scan();
+    return uncertain
+        ? JSON.stringify(["source", source])
+        : JSON.stringify(["tokens", serialize(tokens)]);
+}
+function serialize(tokens) {
+    tokens = normalizePunctuation(tokens);
+    return JSON.stringify(tokens.map((token, index) => {
+        const previous = tokens[index - 1];
+        // Only discard newlines where continuation is certain. Keeping other
+        // newlines permits conservative misses rather than unsafe cache hits.
+        const significant = token.lineBreak && previous !== undefined && (restrictedLineBreaks.has(previous.text) ||
+            ["++", "--"].includes(token.text) ||
+            !((previous.kind === "punctuation" &&
+                (expressionPrefixes.has(previous.text) ||
+                    [".", "?.", "}"].includes(previous.text))) ||
+                (token.kind === "punctuation" &&
+                    ([")", "]", "}", ",", ";", ".", "?."].includes(token.text) ||
+                        (expressionPrefixes.has(token.text) &&
+                            !["(", "[", "{", "!", "~"].includes(token.text))))));
+        return [token.kind, token.text, significant];
+    }));
+}
+function normalizePunctuation(input) {
+    const tokens = input.map((token) => ({ ...token }));
+    const result = [];
+    const brackets = [];
+    const controls = new Set(["if", "while", "for", "with", "switch", "catch"]);
+    let previousControl = false;
+    for (let i = 0; i < tokens.length; i++) {
+        const token = tokens[i];
+        const next = tokens[i + 1];
+        const previous = result.at(-1);
+        if (token.kind === "literal" && /^["']/.test(token.text) &&
+            !token.text.includes("\\")) {
+            token.text = JSON.stringify(token.text.slice(1, -1));
+        }
+        // A single arrow parameter has the same binding with or without parentheses.
+        if (token.text === "(" && tokens[i + 1]?.kind === "word" &&
+            tokens[i + 2]?.text === ")" && tokens[i + 3]?.text === "=>") {
+            result.push({ ...tokens[i + 1], lineBreak: token.lineBreak });
+            i += 2;
+            previousControl = false;
+            continue;
+        }
+        // Retain commas representing array holes, including the final hole.
+        if (token.text === "," && [")", "]", "}"].includes(next?.text ?? "") &&
+            previous && ![",", "[", "("].includes(previous.text))
+            continue;
+        const expressionEnd = previous && (previous.kind !== "punctuation" || previous.text === "]" ||
+            (previous.text === ")" && !previousControl));
+        if (token.text === ";" && brackets.at(-1)?.text === "{" && expressionEnd &&
+            !restrictedLineBreaks.has(previous.text)) {
+            if (next?.text === "}")
+                continue;
+            // A call followed by an identifier starts a new statement. Preserve that
+            // boundary as a newline so explicit and automatically inserted semicolons
+            // agree. Never do this in a for header or after a control header.
+            if (previous.text === ")" && next?.kind === "word" &&
+                !["in", "instanceof", "of"].includes(next.text)) {
+                next.lineBreak = true;
+                continue;
+            }
+        }
+        previousControl = false;
+        if (["(", "[", "{"].includes(token.text)) {
+            brackets.push({
+                text: token.text,
+                control: token.text === "(" && (controls.has(previous?.text ?? "") ||
+                    (previous?.text === "await" && result.at(-2)?.text === "for")),
+            });
+        }
+        else if ([")", "]", "}"].includes(token.text)) {
+            previousControl = brackets.pop()?.control ?? false;
+        }
+        result.push(token);
+    }
+    return result;
+}

package/script/helpers/serializeCacheInputs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"serializeCacheInputs.d.ts","sourceRoot":"","sources":["../../src/helpers/serializeCacheInputs.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,MAAM,CAAC,OAAO,UAAU,oBAAoB,CAC1C,MAAM,EAAE,SAAS,OAAO,EAAE,GACzB,MAAM,CASR;AAED,+EAA+E;AAC/E,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAIrE"}
+{"version":3,"file":"serializeCacheInputs.d.ts","sourceRoot":"","sources":["../../src/helpers/serializeCacheInputs.ts"],"names":[],"mappings":"AAIA;;;GAGG;AACH,MAAM,CAAC,OAAO,UAAU,oBAAoB,CAC1C,MAAM,EAAE,SAAS,OAAO,EAAE,GACzB,MAAM,CASR;AAED,+EAA+E;AAC/E,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAIrE"}

package/script/helpers/serializeCacheInputs.js

@@ -7,6 +7,7 @@ exports.default = serializeCacheInputs;
 exports.getCacheInputLabel = getCacheInputLabel;
 const SimpleTable_js_1 = __importDefault(require("../class/SimpleTable.js"));
 const tableGeneration_js_1 = require("./tableGeneration.js");
+const normalizeCacheCode_js_1 = __importDefault(require("./normalizeCacheCode.js"));
 /**
  * Converts cache inputs to a deterministic, type-tagged representation.
  * SimpleTable values are represented by their name and current generation.
@@ -41,7 +42,7 @@ function serializeValue(value, path, ancestors) {
         case "function":
             return JSON.stringify([
                 "function",
-                Function.prototype.toString.call(value),
+                (0, normalizeCacheCode_js_1.default)(Function.prototype.toString.call(value)),
             ]);
         case "symbol":
             throw new TypeError(`cache() ${path} cannot be a symbol.`);

package/script/methods/cache.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/methods/cache.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AA4DvD,wBAA8B,KAAK,CAAC,KAAK,SAAS,WAAW,EAC3D,KAAK,EAAE,KAAK,EACZ,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAC/C,OAAO,GAAE;IACP,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,iBA4LP"}
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/methods/cache.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AA6DvD,wBAA8B,KAAK,CAAC,KAAK,SAAS,WAAW,EAC3D,KAAK,EAAE,KAAK,EACZ,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAC/C,OAAO,GAAE;IACP,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,iBA4LP"}

package/script/methods/cache.js

@@ -41,6 +41,7 @@ const node_fs_1 = require("node:fs");
 const node_crypto_1 = __importDefault(require("node:crypto"));
 const flushAllTables_js_1 = __importDefault(require("../helpers/flushAllTables.js"));
 const prettyDuration_js_1 = __importDefault(require("../helpers/prettyDuration.js"));
+const normalizeCacheCode_js_1 = __importDefault(require("../helpers/normalizeCacheCode.js"));
 const cacheDiagnostics_js_1 = require("../helpers/cacheDiagnostics.js");
 const serializeCacheInputs_js_1 = __importStar(require("../helpers/serializeCacheInputs.js"));
 const tableGeneration_js_1 = require("../helpers/tableGeneration.js");
@@ -66,7 +67,7 @@ async function cache(table, compute, options = {}) {
     if ((0, node_fs_1.existsSync)(cacheSourcesPath)) {
         cacheSources = JSON.parse((0, node_fs_1.readFileSync)(cacheSourcesPath, "utf-8"));
     }
-    const functionBody = compute.toString();
+    const functionBody = (0, normalizeCacheCode_js_1.default)(compute.toString());
     const entryGeneration = (0, tableGeneration_js_1.peekTableGeneration)(table) ?? null;
     const inputs = options.inputs?.filter((input) => input !== table) ?? [];
     const hasInputs = inputs.length > 0;

package/script/methods/trim.d.ts

@@ -1,5 +1,5 @@
 import type SimpleTable from "../class/SimpleTable.js";
-export default function trim(simpleTable: SimpleTable, columns: string | string[], options?: {
+export default function trim(simpleTable: SimpleTable, columns: "all" | string | string[], options?: {
     character?: string;
     side?: "left" | "right" | "both";
 }): void;

package/script/methods/trim.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"trim.d.ts","sourceRoot":"","sources":["../../src/methods/trim.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,IAAI,CAC1B,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;CAC7B,QAuCP"}
+{"version":3,"file":"trim.d.ts","sourceRoot":"","sources":["../../src/methods/trim.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,MAAM,CAAC,OAAO,UAAU,IAAI,CAC1B,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,KAAK,GAAG,MAAM,GAAG,MAAM,EAAE,EAClC,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;CAC7B,QAgDP"}

package/script/methods/trim.js

@@ -8,7 +8,9 @@ const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier
 const stringToArray_js_1 = __importDefault(require("../helpers/stringToArray.js"));
 const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
 function trim(simpleTable, columns, options = {}) {
-    const cols = (0, stringToArray_js_1.default)(columns);
+    const cols = columns === "all" ? null : (0, stringToArray_js_1.default)(columns);
+    const resolveColumns = (schema) => cols ??
+        Object.keys(schema).filter((column) => schema[column] === "VARCHAR");
     const side = options.side ?? "both";
     const fn = side === "left"
         ? "LTRIM"
@@ -20,17 +22,24 @@ function trim(simpleTable, columns, options = {}) {
     if (fn === null) {
         throw new Error(`trim() options.side must be "left", "right", or "both". Received ${JSON.stringify(options.side)}.`);
     }
-    const specialCharacter = typeof options.character === "string" ? ", ?" : "";
+    const character = options.character;
+    const specialCharacter = typeof character === "string" ? ", ?" : "";
     (0, queueOp_js_1.default)(simpleTable, {
         kind: "fusable",
         method: "trim()",
         parameters: { columns, options },
-        values: typeof options.character === "string"
-            ? cols.map(() => options.character)
+        values: typeof character === "string"
+            ? (schema) => resolveColumns(schema).map(() => character)
             : undefined,
-        needsSchema: false,
-        buildSelect: (input) => `SELECT * REPLACE (${cols
-            .map((c) => `${fn}(${(0, quoteIdentifier_js_1.default)(c)}${specialCharacter}) AS ${(0, quoteIdentifier_js_1.default)(c)}`)
-            .join(", ")}) FROM ${input}`,
+        needsSchema: columns === "all",
+        buildSelect: (input, schema) => {
+            const selectedColumns = resolveColumns(schema);
+            if (columns === "all" && selectedColumns.length === 0) {
+                return `SELECT * FROM ${input}`;
+            }
+            return `SELECT * REPLACE (${selectedColumns
+                .map((c) => `${fn}(${(0, quoteIdentifier_js_1.default)(c)}${specialCharacter}) AS ${(0, quoteIdentifier_js_1.default)(c)}`)
+                .join(", ")}) FROM ${input}`;
+        },
     });
 }