@shrkcrft/compress 0.1.0-alpha.17 → 0.1.0-alpha.18

package/dist/index.d.ts

@@ -25,6 +25,7 @@ export type { IFieldSpec } from './table/field-spec.js';
 export type { ITableCompaction } from './table/table-compaction.js';
 export { compactObjectArray } from './table/compact-object-array.js';
 export type { IColumnarTable } from './table/columnar-table.js';
+export type { IDerivedColumn } from './table/derived-columns.js';
 export { tableToColumnar, compactArrayToColumnar, isColumnarTable, expandColumnar, } from './table/columnar-json.js';
 export { renderTable } from './table/render-table.js';
 export { renderCompactJson } from './json/render-compact-json.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7E,YAAY,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAG/D,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAGzE,YAAY,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtF,YAAY,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,YAAY,EAAE,uBAAuB,EAAE,MAAM,6BAA6B,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAG9D,YAAY,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AACrE,YAAY,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EACL,eAAe,EACf,sBAAsB,EACtB,eAAe,EACf,cAAc,GACf,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAGvD,OAAO,EACL,aAAa,EACb,YAAY,EACZ,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,0BAA0B,CAAC;AAGlC,YAAY,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAGvF,YAAY,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAGnH,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAG5D,YAAY,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,YAAY,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAGnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAC;AACxE,YAAY,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACzE,YAAY,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAGrE,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAG/D,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAGvD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,YAAY,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3E,YAAY,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACjF,YAAY,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAG3E,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7E,YAAY,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAG/D,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAGzE,YAAY,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtF,YAAY,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,YAAY,EAAE,uBAAuB,EAAE,MAAM,6BAA6B,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAG9D,YAAY,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AACrE,YAAY,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,YAAY,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EACL,eAAe,EACf,sBAAsB,EACtB,eAAe,EACf,cAAc,GACf,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAGvD,OAAO,EACL,aAAa,EACb,YAAY,EACZ,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,0BAA0B,CAAC;AAGlC,YAAY,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAGvF,YAAY,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAGnH,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAG5D,YAAY,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,YAAY,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAGnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAC;AACxE,YAAY,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACzE,YAAY,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAGrE,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAG/D,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAGvD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,YAAY,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3E,YAAY,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACjF,YAAY,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAG3E,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC"}

package/dist/table/columnar-json.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"columnar-json.d.ts","sourceRoot":"","sources":["../../src/table/columnar-json.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAI1D;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,gBAAgB,GAAG,cAAc,CAWvE;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG,cAAc,GAAG,IAAI,CAG5E;AAED,wDAAwD;AACxD,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,cAAc,CAYvE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAiCpF"}
+{"version":3,"file":"columnar-json.d.ts","sourceRoot":"","sources":["../../src/table/columnar-json.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAK1D;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,gBAAgB,GAAG,cAAc,CAgBvE;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG,cAAc,GAAG,IAAI,CAG5E;AAED,wDAAwD;AACxD,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,cAAc,CAYvE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAmCpF"}

package/dist/table/columnar-json.js

@@ -1,5 +1,6 @@
 import { compactObjectArray } from "./compact-object-array.js";
 import { applyValueDictionaries } from "./apply-value-dictionaries.js";
+import { dropDerivedColumns, reconstructDerived } from "./derived-columns.js";
 /**
  * Encode a compacted table as a valid-JSON columnar object. Only the fields a
  * decoder needs are emitted — `cols`, `rows`, `absent` (and an optional `dict`).
@@ -8,14 +9,19 @@ import { applyValueDictionaries } from "./apply-value-dictionaries.js";
  * tokens. Low-cardinality columns are value-dictionary encoded (never inflates).
  */
 export function tableToColumnar(table) {
-    const cols = table.cols.map((c) => c.name);
-    const { rows, dict } = applyValueDictionaries(cols, table.rows, table.absent);
+    const cols0 = table.cols.map((c) => c.name);
+    // Drop columns whose every-row value is derivable from another column
+    // (e.g. graph file nodes: id="file:"+path, label=basename(path), kind=const).
+    // Runs on raw values, before value-dictionary encoding.
+    const split = dropDerivedColumns(cols0, table.rows, table.absent);
+    const { rows, dict } = applyValueDictionaries(split.cols, split.rows, split.absent);
     return {
         _table: {
-            cols,
+            cols: split.cols,
             rows,
-            absent: table.absent,
+            absent: split.absent,
             ...(dict ? { dict } : {}),
+            ...(split.derived && split.derived.length > 0 ? { derived: split.derived } : {}),
         },
     };
 }
@@ -47,7 +53,7 @@ export function isColumnarTable(value) {
  * (an absent key stays absent; key order is not significant).
  */
 export function expandColumnar(table) {
-    const { cols, rows, absent, dict } = table._table;
+    const { cols, rows, absent, dict, derived } = table._table;
     const width = cols.length;
     const absentSet = new Set(absent.map(([r, c]) => r * width + c));
     const out = [];
@@ -77,6 +83,9 @@ export function expandColumnar(table) {
                 configurable: true,
             });
         }
+        // Rebuild columns dropped because they were a pure function of a kept one.
+        if (derived && derived.length > 0)
+            reconstructDerived(obj, derived);
         out.push(obj);
     }
     return out;

package/dist/table/columnar-table.d.ts

@@ -5,6 +5,7 @@
  * while still deduplicating the schema out of every row. This is what shrk's
  * MCP tools emit so JSON-parsing agents keep working.
  */
+import type { IDerivedColumn } from './derived-columns.js';
 export interface IColumnarTable {
     _table: {
         /** Column names, in schema order. */
@@ -19,6 +20,12 @@ export interface IColumnarTable {
          * integer indices into `dict[name]` — deref to recover the real value.
          */
         dict?: Record<string, unknown[]>;
+        /**
+         * Optional reconstruction list for columns dropped because each row's value
+         * is a pure function of another column (`const` | `prefix` | `basename`).
+         * `expandColumnar` rebuilds them losslessly; absent here means none dropped.
+         */
+        derived?: IDerivedColumn[];
     };
 }
 //# sourceMappingURL=columnar-table.d.ts.map

package/dist/table/columnar-table.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"columnar-table.d.ts","sourceRoot":"","sources":["../../src/table/columnar-table.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE;QACN,qCAAqC;QACrC,IAAI,EAAE,MAAM,EAAE,CAAC;QACf,0DAA0D;QAC1D,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC;QAClB,wEAAwE;QACxE,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAChC;;;;WAIG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;KAClC,CAAC;CACH"}
+{"version":3,"file":"columnar-table.d.ts","sourceRoot":"","sources":["../../src/table/columnar-table.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE3D,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE;QACN,qCAAqC;QACrC,IAAI,EAAE,MAAM,EAAE,CAAC;QACf,0DAA0D;QAC1D,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC;QAClB,wEAAwE;QACxE,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAChC;;;;WAIG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACjC;;;;WAIG;QACH,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;KAC5B,CAAC;CACH"}

package/dist/table/derived-columns.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Reversible "derived column" pass for the columnar codec.
+ *
+ * Many homogeneous object arrays carry columns that are a pure, deterministic
+ * function of another column FOR EVERY ROW — e.g. a graph file node where
+ * `id === "file:" + path`, `label === basename(path)`, `kind === "file"`.
+ * Value-dictionary/columnar encoding can only dedupe the *key* and
+ * low-cardinality *values*; it can't recover this per-row derivable content.
+ * This pass drops such a column entirely and records how to rebuild it, so
+ * {@link reconstructDerived} restores the exact value on decode.
+ *
+ * Correctness rules:
+ *   - CONTENT-checked over every row (never a key-name heuristic) — a column
+ *     is only dropped when the transform holds for all rows. Rule/path nodes
+ *     where `label` is NOT a function of `id` are left untouched.
+ *   - A derived column's base must itself be irreducible (a "base" column), so
+ *     decode never chases a dropped column. No cycles, no chains.
+ *   - Only fully-present columns (no absent cells) participate, so the `absent`
+ *     map never references a dropped column.
+ */
+export interface IDerivedColumn {
+    /** Column to reconstruct on decode. */
+    name: string;
+    /** Reconstruction op. */
+    op: 'const' | 'prefix' | 'basename';
+    /** Base column name the value is derived from (op = prefix | basename). */
+    from?: string;
+    /** Prefix string (op = prefix) or the constant value (op = const). */
+    arg?: unknown;
+}
+export interface IDerivedSplit {
+    cols: string[];
+    rows: unknown[][];
+    absent: Array<[number, number]>;
+    derived?: IDerivedColumn[];
+}
+/**
+ * Detect and drop derivable columns from a compacted table's raw rows. Returns
+ * the reduced cols/rows/absent plus the `derived` reconstruction list (omitted
+ * when nothing was dropped). Runs BEFORE value-dictionary encoding so it sees
+ * the real values.
+ */
+export declare function dropDerivedColumns(cols: readonly string[], rows: readonly unknown[][], absent: ReadonlyArray<[number, number]>): IDerivedSplit;
+/** Restore the dropped derived columns onto a decoded object (in place). */
+export declare function reconstructDerived(obj: Record<string, unknown>, derived: readonly IDerivedColumn[]): void;
+//# sourceMappingURL=derived-columns.d.ts.map

package/dist/table/derived-columns.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"derived-columns.d.ts","sourceRoot":"","sources":["../../src/table/derived-columns.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,WAAW,cAAc;IAC7B,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,yBAAyB;IACzB,EAAE,EAAE,OAAO,GAAG,QAAQ,GAAG,UAAU,CAAC;IACpC,2EAA2E;IAC3E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sEAAsE;IACtE,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC;IAClB,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAChC,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;CAC5B;AAoBD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,IAAI,EAAE,SAAS,OAAO,EAAE,EAAE,EAC1B,MAAM,EAAE,aAAa,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GACtC,aAAa,CA4Gf;AAED,4EAA4E;AAC5E,wBAAgB,kBAAkB,CAChC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC5B,OAAO,EAAE,SAAS,cAAc,EAAE,GACjC,IAAI,CAiBN"}

package/dist/table/derived-columns.js

@@ -0,0 +1,172 @@
+/**
+ * Reversible "derived column" pass for the columnar codec.
+ *
+ * Many homogeneous object arrays carry columns that are a pure, deterministic
+ * function of another column FOR EVERY ROW — e.g. a graph file node where
+ * `id === "file:" + path`, `label === basename(path)`, `kind === "file"`.
+ * Value-dictionary/columnar encoding can only dedupe the *key* and
+ * low-cardinality *values*; it can't recover this per-row derivable content.
+ * This pass drops such a column entirely and records how to rebuild it, so
+ * {@link reconstructDerived} restores the exact value on decode.
+ *
+ * Correctness rules:
+ *   - CONTENT-checked over every row (never a key-name heuristic) — a column
+ *     is only dropped when the transform holds for all rows. Rule/path nodes
+ *     where `label` is NOT a function of `id` are left untouched.
+ *   - A derived column's base must itself be irreducible (a "base" column), so
+ *     decode never chases a dropped column. No cycles, no chains.
+ *   - Only fully-present columns (no absent cells) participate, so the `absent`
+ *     map never references a dropped column.
+ */
+/** Last path segment (everything after the final `/`), or the whole string. */
+function basename(s) {
+    const i = s.lastIndexOf('/');
+    return i === -1 ? s : s.slice(i + 1);
+}
+function copy(cols, rows, absent) {
+    return {
+        cols: [...cols],
+        rows: rows.map((r) => [...r]),
+        absent: absent.map((a) => [a[0], a[1]]),
+    };
+}
+/**
+ * Detect and drop derivable columns from a compacted table's raw rows. Returns
+ * the reduced cols/rows/absent plus the `derived` reconstruction list (omitted
+ * when nothing was dropped). Runs BEFORE value-dictionary encoding so it sees
+ * the real values.
+ */
+export function dropDerivedColumns(cols, rows, absent) {
+    const n = rows.length;
+    // Multi-row only — a single row never pays for the reconstruction metadata.
+    if (n < 2 || cols.length < 1)
+        return copy(cols, rows, absent);
+    const absentCols = new Set();
+    for (const [, c] of absent)
+        absentCols.add(c);
+    const present = (c) => !absentCols.has(c);
+    const val = (r, c) => rows[r][c];
+    const isStringCol = (c) => present(c) && rows.every((row) => typeof row[c] === 'string');
+    const dropped = new Set();
+    const derived = [];
+    // 1) Constant columns (every present row holds the same JSON primitive).
+    for (let c = 0; c < cols.length; c += 1) {
+        if (!present(c))
+            continue;
+        const first = val(0, c);
+        if (typeof first === 'object' && first !== null)
+            continue; // only primitives/null
+        let allEqual = true;
+        for (let r = 1; r < n; r += 1) {
+            if (val(r, c) !== first) {
+                allEqual = false;
+                break;
+            }
+        }
+        if (allEqual) {
+            dropped.add(c);
+            derived.push({ name: cols[c], op: 'const', arg: first });
+        }
+    }
+    // 2) Prefix / basename derivations among the remaining string columns.
+    const stringCols = [];
+    for (let c = 0; c < cols.length; c += 1) {
+        if (!dropped.has(c) && isStringCol(c))
+            stringCols.push(c);
+    }
+    // All valid prefix/basename derivations of column `c` (from every other
+    // string column), so we can later prefer one whose base is irreducible.
+    const allTransformsOf = (c) => {
+        const out = [];
+        for (const d of stringCols) {
+            if (d === c)
+                continue;
+            const c0 = val(0, c);
+            const d0 = val(0, d);
+            // prefix: c === arg + d for all rows (c contains d as a suffix).
+            if (c0.endsWith(d0) && c0.length > d0.length) {
+                const arg = c0.slice(0, c0.length - d0.length);
+                let ok = true;
+                for (let r = 0; r < n; r += 1) {
+                    if (val(r, c) !== arg + val(r, d)) {
+                        ok = false;
+                        break;
+                    }
+                }
+                if (ok)
+                    out.push({ name: cols[c], op: 'prefix', from: cols[d], arg });
+            }
+            // basename: c === basename(d) for all rows.
+            let okB = true;
+            for (let r = 0; r < n; r += 1) {
+                if (val(r, c) !== basename(val(r, d))) {
+                    okB = false;
+                    break;
+                }
+            }
+            if (okB)
+                out.push({ name: cols[c], op: 'basename', from: cols[d] });
+        }
+        return out;
+    };
+    const candidates = new Map();
+    for (const c of stringCols)
+        candidates.set(c, allTransformsOf(c));
+    // A column is a BASE iff it cannot be derived from any other column.
+    const baseNames = new Set();
+    for (const c of stringCols)
+        if (candidates.get(c).length === 0)
+            baseNames.add(cols[c]);
+    // Drop a column only when it can be derived from an irreducible base, so the
+    // decoder never chases a dropped column (breaks mutual-derivability cleanly:
+    // path stays, id & label derive from it).
+    for (const c of stringCols) {
+        const pick = candidates.get(c).find((t) => t.from && baseNames.has(t.from));
+        if (pick) {
+            dropped.add(c);
+            derived.push(pick);
+        }
+    }
+    if (dropped.size === 0)
+        return copy(cols, rows, absent);
+    // Rebuild cols/rows without the dropped columns; remap `absent` col indices
+    // (dropped columns carry no absent cells, so none are lost).
+    const oldToNew = new Map();
+    const keptIdx = [];
+    for (let c = 0; c < cols.length; c += 1) {
+        if (!dropped.has(c)) {
+            oldToNew.set(c, keptIdx.length);
+            keptIdx.push(c);
+        }
+    }
+    const newCols = keptIdx.map((c) => cols[c]);
+    const newRows = rows.map((row) => keptIdx.map((c) => row[c]));
+    const newAbsent = [];
+    for (const [r, c] of absent) {
+        const nc = oldToNew.get(c);
+        if (nc !== undefined)
+            newAbsent.push([r, nc]);
+    }
+    return { cols: newCols, rows: newRows, absent: newAbsent, derived };
+}
+/** Restore the dropped derived columns onto a decoded object (in place). */
+export function reconstructDerived(obj, derived) {
+    for (const d of derived) {
+        let value;
+        if (d.op === 'const') {
+            value = d.arg;
+        }
+        else {
+            const base = obj[d.from];
+            if (typeof base !== 'string')
+                continue; // base missing/non-string — leave absent
+            value = d.op === 'prefix' ? String(d.arg) + base : basename(base);
+        }
+        Object.defineProperty(obj, d.name, {
+            value,
+            writable: true,
+            enumerable: true,
+            configurable: true,
+        });
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shrkcrft/compress",
-  "version": "0.1.0-alpha.17",
+  "version": "0.1.0-alpha.18",
   "description": "SharkCraft deterministic context-compression engine: content routing, lossless columnar/table compaction, log/search/diff line reduction, and reversible Compress-Cache-Retrieve (CCR). No model inside — every transform is a pure function of its input.",
   "license": "MIT",
   "author": "SharkCraft contributors",
@@ -44,7 +44,7 @@
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@shrkcrft/core": "^0.1.0-alpha.17"
+    "@shrkcrft/core": "^0.1.0-alpha.18"
   },
   "publishConfig": {
     "access": "public"