@nshiab/simple-data-analysis-core 2.1.12 → 2.1.13

package/src/helpers/cacheTableDependencies.ts

@@ -0,0 +1,292 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import referencedTables from "./referencedTables.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import type { PendingOp, PendingOpInput } from "./pendingOps.ts";
+import {
+  getAsyncOperationFrame,
+  runWhileDrainingAsyncOperationFrame,
+} from "./asyncOperationContext.ts";
+import flushAllTables, { runExemptFromFlush } from "./flushAllTables.ts";
+import quoteIdentifier from "./quoteIdentifier.ts";
+import { getRegisteredTables, retainRegisteredTables } from "./tableRegistry.ts";
+import {
+  getTableGeneration,
+  peekTableGeneration,
+  type TableGenerationId,
+} from "./tableGeneration.ts";
+
+export type CacheTableDependency = {
+  tableName: string;
+  generationId: TableGenerationId;
+};
+
+type CacheTableSnapshot = {
+  tableName: string;
+  generationId: TableGenerationId | undefined;
+};
+
+type CacheComputation = {
+  cachedTable: SimpleTable;
+  availableBeforeCompute: Map<SimpleTable, CacheTableSnapshot>;
+  accessed: Set<SimpleTable>;
+  created: Set<SimpleTable>;
+  queued: Set<number>;
+  open: boolean;
+};
+
+const dependencyContext = new AsyncLocalStorage<CacheComputation[]>();
+
+/** Keeps deferred callbacks in their originating cache scope across observers. */
+export function captureCacheTableOperation(op: PendingOpInput): PendingOpInput {
+  if (op.kind === "barrier" || op.kind === "asyncBarrier") {
+    return { ...op, execute: bindCacheContext(op.execute) };
+  }
+  if (op.kind === "source") {
+    return { ...op, buildSelect: bindCacheContext(op.buildSelect) };
+  }
+  return {
+    ...op,
+    buildSelect: bindCacheContext(op.buildSelect),
+    outputSchema: op.outputSchema === undefined ? undefined : bindCacheContext(op.outputSchema),
+    values: typeof op.values === "function" ? bindCacheContext(op.values) : op.values,
+  };
+}
+
+function bindCacheContext<Args extends unknown[], Result>(
+  callback: (...args: Args) => Result,
+): (...args: Args) => Result {
+  // Capture only the cache scope: a snapshot of all AsyncLocalStorage stores
+  // would also capture stale flush exemptions and async operation frames.
+  // An empty scope keeps unrelated work out of the observer's cache callback.
+  const frames = dependencyContext.getStore() ?? [];
+  return (...args) => dependencyContext.run(frames, () => callback(...args));
+}
+
+/** Rejects mutations before they can queue work or execute SQL. */
+export function assertCacheTableMutation(table: SimpleTable): void {
+  for (const frame of dependencyContext.getStore() ?? []) {
+    if (frame.open && table !== frame.cachedTable && frame.availableBeforeCompute.has(table)) {
+      throw new Error(cacheTableMutationMessage(frame.cachedTable, [table.name]));
+    }
+  }
+}
+
+/** Tracks callback-owned handles so failure cleanup does not touch other work. */
+export function recordCacheTableCreation(table: SimpleTable): void {
+  for (const frame of dependencyContext.getStore() ?? []) {
+    if (
+      frame.open &&
+      table.sdb === frame.cachedTable.sdb &&
+      !frame.availableBeforeCompute.has(table)
+    ) {
+      frame.created.add(table);
+    }
+  }
+}
+
+/** Tracks queued work across nested callbacks, including async barriers. */
+export function recordCacheTableOperation(table: SimpleTable, op: PendingOp): void {
+  for (const frame of dependencyContext.getStore() ?? []) {
+    if (frame.open && table.sdb === frame.cachedTable.sdb) {
+      frame.queued.add(op.sequence);
+    }
+  }
+}
+
+/** Records a table read when a cache computation is observing dependencies. */
+export function recordCacheTableAccess(table: SimpleTable): void {
+  for (const frame of dependencyContext.getStore() ?? []) {
+    if (frame.open && table.sdb === frame.cachedTable.sdb) {
+      frame.accessed.add(table);
+    }
+  }
+}
+
+/** Records registered tables named in user-supplied SQL fragments. */
+export function recordCacheTableReferences(table: SimpleTable, sql: readonly string[]): void {
+  if (dependencyContext.getStore() === undefined) {
+    return;
+  }
+  for (const candidate of referencedTables(sql, table.sdb)) {
+    recordCacheTableAccess(candidate);
+  }
+}
+
+/** Runs a cache computation while collecting the tables it reads. */
+export async function captureCacheTableDependencies(
+  cachedTable: SimpleTable,
+  compute: () => Promise<void>,
+): Promise<CacheTableDependency[]> {
+  const availableBeforeCompute = new Map(
+    getRegisteredTables(cachedTable.sdb).map((table) => [
+      table,
+      {
+        tableName: table.name,
+        generationId: peekTableGeneration(table),
+      },
+    ]),
+  );
+  const accessed = new Set<SimpleTable>();
+  const frame: CacheComputation = {
+    cachedTable,
+    availableBeforeCompute,
+    accessed,
+    created: new Set(),
+    queued: new Set(),
+    open: true,
+  };
+  try {
+    await dependencyContext.run([...(dependencyContext.getStore() ?? []), frame], async () => {
+      await compute();
+      // A callback can queue a whole computation without observing it.
+      // Reject leaked handles before executing that pending computation.
+      assertCacheTableCleanup(frame);
+      await flushAllTables(cachedTable.sdb);
+      // Async barriers may create tables during the final flush.
+      assertCacheTableCleanup(frame);
+    });
+  } catch (error) {
+    await cleanupCacheComputation(frame);
+    throw error;
+  } finally {
+    frame.open = false;
+  }
+
+  return [...accessed]
+    .filter((table) => table !== cachedTable && availableBeforeCompute.has(table))
+    .map((table) => {
+      const snapshot = availableBeforeCompute.get(table);
+      if (snapshot === undefined) {
+        throw new Error("A cache table dependency snapshot is missing.");
+      }
+      return {
+        tableName: snapshot.tableName,
+        generationId: snapshot.generationId ?? getTableGeneration(table),
+      };
+    })
+    .sort((left, right) => left.tableName.localeCompare(right.tableName));
+}
+
+function assertCacheTableCleanup(frame: CacheComputation): void {
+  const created = getRegisteredTables(frame.cachedTable.sdb).filter((table) =>
+    frame.created.has(table),
+  );
+  if (created.length > 0) {
+    throw new Error(cacheTableCleanupMessage(created.map((table) => table.name)));
+  }
+}
+
+async function cleanupCacheComputation(frame: CacheComputation): Promise<void> {
+  const sdb = frame.cachedTable.sdb;
+  // Drop only this callback's pending operations, preserving other queues and
+  // their database-wide ordering. A nested failure also discards work captured
+  // by an enclosing async barrier, which is stored outside pendingOps.
+  for (const table of getRegisteredTables(sdb)) {
+    const retained = table.pendingOps.filter((op) => !frame.queued.has(op.sequence));
+    sdb.pendingCount -= table.pendingOps.length - retained.length;
+    table.pendingOps.splice(0, table.pendingOps.length, ...retained);
+  }
+  const operationFrame = getAsyncOperationFrame(sdb);
+  if (operationFrame !== undefined) {
+    operationFrame.entries = operationFrame.entries.filter(
+      (entry) => !frame.queued.has(entry.op.sequence),
+    );
+  }
+  const created = getRegisteredTables(sdb).filter((table) => frame.created.has(table));
+  if (created.length === 0) return;
+  // Observers inside compute may already have materialized temporary tables.
+  // Cleanup must not flush work requeued after an execution error.
+  const removeCreatedTables = () =>
+    runExemptFromFlush(sdb, async () => {
+      for (const table of created) {
+        await sdb.customQuery(`DROP TABLE IF EXISTS ${quoteIdentifier(table.name)};`);
+      }
+    });
+  if (operationFrame === undefined) {
+    await removeCreatedTables();
+  } else {
+    // Async frames take precedence over the database-wide flush exemption.
+    // Suppress their drain too, without consuming unrelated captured work.
+    await runWhileDrainingAsyncOperationFrame(operationFrame, removeCreatedTables);
+  }
+  retainRegisteredTables(sdb, (table) => !frame.created.has(table));
+}
+
+function cacheTableMutationMessage(
+  cachedTable: SimpleTable,
+  changedTables: readonly string[],
+): string {
+  const tables = changedTables.map((name) => JSON.stringify(name)).join(", ");
+  const cachedTableName = JSON.stringify(cachedTable.name);
+  if (changedTables.length === 1) {
+    return `cache() called on ${cachedTableName} cannot modify pre-existing table ${tables}. Modify ${tables} outside this cache() call.`;
+  }
+  return `cache() called on ${cachedTableName} cannot modify pre-existing tables ${tables}. Modify these tables outside this cache() call.`;
+}
+
+function cacheTableCleanupMessage(createdTables: readonly string[]): string {
+  const tables = createdTables.map((name) => JSON.stringify(name)).join(", ");
+  if (createdTables.length === 1) {
+    return `cache() created table ${tables} but did not remove it. Call removeTable() on ${tables} before the callback finishes to avoid downstream errors when the cache is loaded and this table is not recreated.`;
+  }
+  return `cache() created tables ${tables} but did not remove them. Call removeTable() on these tables before the callback finishes to avoid downstream errors when the cache is loaded and these tables are not recreated.`;
+}
+
+/** Checks persisted dependencies against the current registered tables. */
+export function cacheTableDependenciesMatch(
+  cachedTable: SimpleTable,
+  dependencies: readonly CacheTableDependency[] | undefined,
+): boolean {
+  if (dependencies === undefined) {
+    return false;
+  }
+  const registered = getRegisteredTables(cachedTable.sdb);
+  return dependencies.every((dependency) => {
+    const table = registered.find((candidate) => candidate.name === dependency.tableName);
+    return table !== undefined && getTableGeneration(table) === dependency.generationId;
+  });
+}
+
+/** Describes changed or unavailable table dependencies for verbose logging. */
+export function changedCacheTableDependenciesMessage(
+  cachedTable: SimpleTable,
+  dependencies: readonly CacheTableDependency[],
+): string {
+  const registered = getRegisteredTables(cachedTable.sdb);
+  const changed: CacheTableDependency[] = [];
+  const unchanged: CacheTableDependency[] = [];
+  for (const dependency of dependencies) {
+    const table = registered.find((candidate) => candidate.name === dependency.tableName);
+    if (table === undefined || getTableGeneration(table) !== dependency.generationId) {
+      changed.push(dependency);
+    } else {
+      unchanged.push(dependency);
+    }
+  }
+
+  return [
+    cacheTableDependencyStatusMessage("changed", changed),
+    cacheTableDependencyStatusMessage("unchanged", unchanged),
+  ]
+    .filter((message) => message.length > 0)
+    .join("\n");
+}
+
+/** Lists unchanged dependencies for cache-hit verbose logging. */
+export function unchangedCacheTableDependenciesMessage(
+  dependencies: readonly CacheTableDependency[],
+): string {
+  return cacheTableDependencyStatusMessage("unchanged", dependencies);
+}
+
+function cacheTableDependencyStatusMessage(
+  status: "changed" | "unchanged",
+  dependencies: readonly CacheTableDependency[],
+): string {
+  if (dependencies.length === 0) {
+    return "";
+  }
+  return `Table dependencies ${status}: ${dependencies
+    .map((dependency) => JSON.stringify(dependency.tableName))
+    .join(", ")}.`;
+}

package/src/helpers/camelCase.ts

@@ -0,0 +1,50 @@
+import capitalize from "./capitalize.ts";
+
+/**
+ * Converts a string into camelCase. This is useful for creating variable
+ * names or object keys from human-readable text.
+ *
+ * @example
+ * ```ts
+ * // Basic conversion
+ * const result1 = camelCase("hello world");
+ * console.log(result1); // "helloWorld"
+ * ```
+ *
+ * @example
+ * ```ts
+ * // With punctuation and mixed case
+ * const result2 = camelCase("  --Some@Thing is- happening--  ");
+ * console.log(result2); // "someThingIsHappening"
+ * ```
+ *
+ * @example
+ * ```ts
+ * // With a single word
+ * const result3 = camelCase("Journalism");
+ * console.log(result3); // "journalism"
+ * ```
+ *
+ * @param input - The string to convert to camelCase. It can contain spaces,
+ *   punctuation, and mixed casing.
+ *
+ * @returns The camelCased version of the input string.
+ */
+export default function camelCase(input: string): string {
+  const words = input
+    .replace(/[^a-zA-Z0-9\s]/g, " ")
+    .split(/\s+/)
+    .filter((d) => d !== "");
+
+  const camelCaseString = words
+    .map((word, index) => {
+      word = word.toLowerCase();
+      if (index > 0) {
+        word = capitalize(word);
+      }
+      return word;
+    })
+    .join("");
+
+  return camelCaseString;
+}

package/src/helpers/capitalize.ts

@@ -0,0 +1,35 @@
+/**
+ * Capitalizes the first letter of a given string.
+ *
+ * @example
+ * ```ts
+ * // Basic usage
+ * const capitalized = capitalize("hello world");
+ * console.log(capitalized); // "Hello world"
+ * ```
+ *
+ * @example
+ * ```ts
+ * // With an already capitalized string
+ * const alreadyCapitalized = capitalize("Journalism");
+ * console.log(alreadyCapitalized); // "Journalism"
+ * ```
+ *
+ * @example
+ * ```ts
+ * // With a single character
+ * const singleChar = capitalize("a");
+ * console.log(singleChar); // "A"
+ * ```
+ *
+ * @param string - The string to be capitalized.
+ *
+ * @returns A new string with the first letter in uppercase.
+ */
+export default function capitalize(string: string): string {
+  if (string.length === 0) {
+    return string;
+  }
+  const first = string[0];
+  return first.toUpperCase() + string.slice(1);
+}

package/src/helpers/cleanCache.ts

@@ -0,0 +1,33 @@
+import { readFileSync, unlinkSync, writeFileSync } from "node:fs";
+import type SimpleDB from "../class/SimpleDB.ts";
+
+export default function cleanCache(sdb: SimpleDB, log: (message: string) => void = console.log) {
+  if (sdb.cacheSourcesUsed.length > 0) {
+    const cacheSources: unknown = JSON.parse(readFileSync(".sda-cache/sources.json", "utf8"));
+    if (cacheSources === null || typeof cacheSources !== "object" || Array.isArray(cacheSources)) {
+      throw new Error("Invalid SDA table cache metadata.");
+    }
+    for (const [cacheId, source] of Object.entries(cacheSources)) {
+      if (!isFileSource(source)) throw new Error("Invalid SDA cache file metadata.");
+      if (!sdb.cacheSourcesUsed.includes(cacheId)) {
+        if (source.file !== null) {
+          if (sdb.cacheVerbose) {
+            log(`Removing unused file from cache: ${source.file}`);
+          }
+          unlinkSync(source.file);
+        }
+        Reflect.deleteProperty(cacheSources, cacheId);
+      }
+    }
+    writeFileSync(".sda-cache/sources.json", JSON.stringify(cacheSources));
+  }
+}
+
+function isFileSource(value: unknown): value is { file: string | null } {
+  return (
+    value !== null &&
+    typeof value === "object" &&
+    "file" in value &&
+    (value.file === null || typeof value.file === "string")
+  );
+}

package/src/helpers/cleanPath.ts

@@ -0,0 +1,3 @@
+export default function cleanPath(file: string) {
+  return file.replaceAll("'", "''");
+}

package/src/helpers/cleanSQL.ts

@@ -0,0 +1,278 @@
+import scanSQL from "./scanSQL.ts";
+
+export default function cleanSQL(query: string, syntax: "js" | "sql" = "js"): string {
+  if (syntax === "sql") return query;
+  const tokens = scanSQL(query);
+  const significant = tokens.filter((token) => token.kind !== "trivia");
+  const original = significant.map((token) =>
+    token.kind === "sql" ? token.text.toUpperCase() : "",
+  );
+  const pairs = new Map<number, number>();
+  const stack: number[] = [];
+  for (let i = 0; i < original.length; i++) {
+    if (["(", "[", "CASE"].includes(original[i])) stack.push(i);
+    else if (
+      stack.length &&
+      ((original[i] === ")" && original[stack[stack.length - 1]] === "(") ||
+        (original[i] === "]" && original[stack[stack.length - 1]] === "[") ||
+        (original[i] === "END" && original[stack[stack.length - 1]] === "CASE"))
+    ) {
+      const start = stack.pop()!;
+      pairs.set(start, i);
+      pairs.set(i, start);
+    }
+  }
+  const betweenAnd = new Set<number>();
+  for (let i = 0; i < original.length; i++) {
+    if (original[i] !== "BETWEEN") continue;
+    for (let j = i + 1; j < original.length; j++) {
+      if (["(", "[", "CASE"].includes(original[j])) {
+        j = pairs.get(j) ?? original.length;
+      } else if ([")", "]", "END", ";"].includes(original[j])) break;
+      else if (original[j] === "AND") {
+        betweenAnd.add(j);
+        break;
+      }
+    }
+  }
+  const comparisons = new Set([
+    "=",
+    "==",
+    "===",
+    "!=",
+    "!==",
+    "<>",
+    "<",
+    ">",
+    "<=",
+    ">=",
+    "IS",
+    "IN",
+    "LIKE",
+    "ILIKE",
+    "BETWEEN",
+  ]);
+  const boundaries = new Set([
+    "SELECT",
+    "FROM",
+    "WHERE",
+    "HAVING",
+    "QUALIFY",
+    "GROUP",
+    "ORDER",
+    "LIMIT",
+    "RETURNING",
+    "UNION",
+    "EXCEPT",
+    "INTERSECT",
+    "WHEN",
+    "THEN",
+    "ELSE",
+    "AS",
+    "AND",
+    "OR",
+    "&&",
+    ",",
+    ";",
+  ]);
+  const textTypes = new Set(["VARCHAR", "CHAR", "BPCHAR", "TEXT", "STRING"]);
+
+  function range(
+    pipe: number,
+    start: number,
+    end: number,
+    stop: (i: number) => boolean,
+  ): [number, number] {
+    let left = pipe - 1;
+    while (left >= start && !stop(left)) {
+      if ([")", "]", "END"].includes(original[left])) {
+        left = pairs.get(left) ?? start;
+      }
+      left--;
+    }
+    let right = pipe + 1;
+    while (right < end && !stop(right)) {
+      if (["(", "[", "CASE"].includes(original[right])) {
+        right = pairs.get(right) ?? end;
+      }
+      right++;
+    }
+    return [left + 1, right];
+  }
+
+  function predicate(start: number, end: number): boolean {
+    if (original[start] === "(" && pairs.get(start) === end - 1) {
+      return predicate(start + 1, end - 1);
+    }
+    for (let i = start; i < end; i++) {
+      if (["(", "[", "CASE"].includes(original[i])) i = pairs.get(i) ?? end;
+      else if (comparisons.has(original[i])) return true;
+    }
+    return false;
+  }
+
+  // Only infer types that are explicit in the expression. Never infer a
+  // column's or an arbitrary function's type from its name or argument types.
+  function isTextType(start: number, end: number): boolean {
+    return (
+      textTypes.has(original[start]) &&
+      (start + 1 === end || (original[start + 1] === "(" && pairs.get(start + 1) === end - 1))
+    );
+  }
+
+  function concatenable(start: number, end: number): boolean {
+    if (start >= end) return false;
+    let cast: number | undefined;
+    let pipe: number | undefined;
+    for (let i = start; i < end; i++) {
+      if (["(", "[", "CASE"].includes(original[i])) i = pairs.get(i) ?? end;
+      else if (comparisons.has(original[i])) return false;
+      else if (original[i] === "||") pipe ??= i;
+      else if (original[i] === "::") cast = i;
+    }
+    if (pipe !== undefined) {
+      return concatenable(start, pipe) || concatenable(pipe + 1, end);
+    }
+    // The final cast determines the result type, including casts to BOOLEAN.
+    if (cast !== undefined) {
+      return isTextType(cast + 1, end);
+    }
+    if (original[start] === "(" && pairs.get(start) === end - 1) {
+      return concatenable(start + 1, end - 1);
+    }
+    if (original[start] === "[" && pairs.get(start) === end - 1) return true;
+    if (end === start + 1) {
+      return significant[start].kind === "quoted" && !significant[start].text.startsWith('"');
+    }
+    if (original[start + 1] === "(" && pairs.get(start + 1) === end - 1) {
+      if (["CONCAT", "CONCAT_WS"].includes(original[start])) return true;
+      if (["CAST", "TRY_CAST"].includes(original[start])) {
+        for (let i = start + 2; i < end - 1; i++) {
+          if (["(", "[", "CASE"].includes(original[i])) {
+            i = pairs.get(i) ?? end;
+          } else if (original[i] === "AS") {
+            return isTextType(i + 1, end - 1);
+          }
+        }
+      }
+    }
+    return false;
+  }
+
+  function obviousConcatenation(pipe: number, start: number, end: number): boolean {
+    const [left, right] = range(
+      pipe,
+      start,
+      end,
+      (i) => (boundaries.has(original[i]) && !betweenAnd.has(i)) || original[i] === "||",
+    );
+    // A literal in a comparison is not evidence that the comparison returns text.
+    if (predicate(left, pipe) && predicate(pipe + 1, right)) return false;
+    const [groupStart, groupEnd] = range(
+      pipe,
+      start,
+      end,
+      (i) => boundaries.has(original[i]) || comparisons.has(original[i]),
+    );
+    let operand = groupStart;
+    for (let i = groupStart; i <= groupEnd; i++) {
+      if (["(", "[", "CASE"].includes(original[i])) {
+        i = pairs.get(i) ?? groupEnd;
+      } else if (i === groupEnd || original[i] === "||") {
+        // Do not use the RHS literal of a preceding comparison to infer OR's
+        // meaning (text = 'a' || active). Other operands can still prove concat.
+        if (
+          !(
+            operand === groupStart &&
+            (comparisons.has(original[groupStart - 1]) || betweenAnd.has(groupStart - 1))
+          ) &&
+          concatenable(operand, i)
+        )
+          return true;
+        operand = i + 1;
+      }
+    }
+    return false;
+  }
+
+  function normalize(start: number, end: number) {
+    let statement: string | undefined = original[start];
+    let inSetClause = false;
+    let assignmentTarget = false;
+    for (let i = start; i < end; i++) {
+      const word = original[i];
+      if (statement === undefined) statement = word;
+      // Nested queries are visited separately below. Look through statement
+      // prefixes to find the outer command that can introduce assignments.
+      if (
+        ["WITH", "EXPLAIN", "PREPARE"].includes(statement) &&
+        ["SELECT", "INSERT", "UPDATE", "DELETE", "MERGE"].includes(word)
+      )
+        statement = word;
+      if (word === "UPDATE" && ["DO", "THEN"].includes(original[i - 1])) statement = word;
+      if (["(", "[", "CASE"].includes(word)) {
+        const close = pairs.get(i);
+        if (close !== undefined) {
+          normalize(i + 1, close);
+          i = close;
+          continue;
+        }
+      }
+      if (["WHERE", "HAVING", "QUALIFY", "WHEN"].includes(word)) {
+        statement = word;
+        inSetClause = false;
+        assignmentTarget = false;
+      } else if (
+        [
+          "THEN",
+          "ELSE",
+          "SELECT",
+          "FROM",
+          "GROUP",
+          "ORDER",
+          "LIMIT",
+          "RETURNING",
+          "UNION",
+          "EXCEPT",
+          "INTERSECT",
+          ";",
+        ].includes(word)
+      ) {
+        statement = word;
+        inSetClause = false;
+        assignmentTarget = false;
+      } else if (
+        word === "SET" &&
+        !inSetClause &&
+        (statement === "UPDATE" ||
+          (statement === "SET" && (original[i + 1] === "VARIABLE" || original[i + 2] === "=")))
+      ) {
+        inSetClause = true;
+        assignmentTarget = true;
+      } else if (word === "," && inSetClause) assignmentTarget = true;
+      if (word === ";") statement = undefined;
+
+      // Preserve only the assignment operator itself. Comparisons in its RHS
+      // still use shorthand, even when they are not wrapped in parentheses.
+      const assignment = assignmentTarget && ["=", "==", "==="].includes(word);
+      if (assignment) assignmentTarget = false;
+
+      const token = significant[i];
+      if (["==", "==="].includes(word)) token.text = "=";
+      else if (word === "!==") token.text = "!=";
+      else if (word === "&&") token.text = " AND ";
+      else if (word === "||" && !obviousConcatenation(i, start, end)) {
+        token.text = " OR ";
+      }
+      if (original[i + 1] === "NULL" && !assignment) {
+        if (["=", "==", "==="].includes(word)) token.text = " IS ";
+        else if (["!=", "!=="].includes(word)) token.text = " IS NOT ";
+      }
+    }
+  }
+  normalize(0, significant.length);
+  return tokens
+    .map((token) => token.text)
+    .join("")
+    .trim();
+}