@claude-flow/cli 3.10.15 → 3.10.16

package/dist/src/memory/structured-distill.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Structured Distillation for trajectory content (#2241 §SOTA — arXiv:2603.13017).
+ *
+ * The arXiv paper compresses agent exchanges from ~371 to ~38 tokens (11×) using
+ * a four-field schema, and reports retrieval MRR going from 0.745 (raw) to
+ * 0.759 (distilled) on a 214 K-pair consensus-graded corpus. The schema is
+ * domain-portable; we adopt it for trajectory step content so SONA's recall
+ * works against a denser, higher-signal representation.
+ *
+ * This module is the RULE-BASED extractor. A future round can plug in a
+ * learned distiller (cross-encoder / LLM); the schema and the harness stay
+ * the same so the swap is drop-in.
+ *
+ * Schema:
+ *   summary  — first sentence (capped); the headline of the exchange
+ *   detail   — the rest of the content (capped); kept for fidelity
+ *   labels   — domain tokens: verbs, nouns, and recognised actions
+ *   paths    — file paths and file:line references (high-signal anchors)
+ */
+/** Distilled trajectory content. */
+export interface DistilledContent {
+    summary: string;
+    detail: string;
+    labels: string[];
+    paths: string[];
+}
+/**
+ * Distill a raw trajectory step's content into the 4-field schema.
+ * Deterministic, dependency-free, sub-millisecond.
+ */
+export declare function distillTrajectoryContent(raw: string): DistilledContent;
+/**
+ * Serialise a distilled object into a compact embedding-ready string. The
+ * resulting form is what gets embedded for retrieval — labels and paths come
+ * first so they get high attention weight in the embedding.
+ *
+ * The paper's MRR uplift comes from the ordering: high-signal tokens (labels,
+ * paths) lead, so the embedder allocates more probability mass to them.
+ */
+export declare function serialiseDistilled(d: DistilledContent): string;
+/** Convenience: distill then serialise. */
+export declare function distillAndSerialise(raw: string): string;
+/**
+ * Compression ratio: raw bytes / distilled bytes. >1 means distilled is
+ * smaller. Used by the benchmark harness.
+ */
+export declare function compressionRatio(raw: string): number;
+//# sourceMappingURL=structured-distill.d.ts.map

package/dist/src/memory/structured-distill.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"structured-distill.d.ts","sourceRoot":"","sources":["../../../src/memory/structured-distill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,oCAAoC;AACpC,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AA2DD;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,GAAG,gBAAgB,CAYtE;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAO9D;AAED,2CAA2C;AAC3C,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAIpD"}

package/dist/src/memory/structured-distill.js

@@ -0,0 +1,125 @@
+/**
+ * Structured Distillation for trajectory content (#2241 §SOTA — arXiv:2603.13017).
+ *
+ * The arXiv paper compresses agent exchanges from ~371 to ~38 tokens (11×) using
+ * a four-field schema, and reports retrieval MRR going from 0.745 (raw) to
+ * 0.759 (distilled) on a 214 K-pair consensus-graded corpus. The schema is
+ * domain-portable; we adopt it for trajectory step content so SONA's recall
+ * works against a denser, higher-signal representation.
+ *
+ * This module is the RULE-BASED extractor. A future round can plug in a
+ * learned distiller (cross-encoder / LLM); the schema and the harness stay
+ * the same so the swap is drop-in.
+ *
+ * Schema:
+ *   summary  — first sentence (capped); the headline of the exchange
+ *   detail   — the rest of the content (capped); kept for fidelity
+ *   labels   — domain tokens: verbs, nouns, and recognised actions
+ *   paths    — file paths and file:line references (high-signal anchors)
+ */
+const SUMMARY_MAX = 200;
+const DETAIL_MAX = 1024;
+// Imperative verbs typical of code-work trajectories (Reflexion / ReasoningBank
+// vocabulary). Order-preserving for stable deduplication.
+const ACTION_VOCAB = [
+    'refactor', 'fix', 'add', 'remove', 'rename', 'extract', 'inline',
+    'split', 'merge', 'delete', 'rewrite', 'simplify', 'optimize',
+    'test', 'mock', 'stub', 'verify', 'assert', 'expect',
+    'document', 'explain', 'comment',
+    'debug', 'trace', 'log', 'diagnose',
+    'wire', 'route', 'dispatch', 'register', 'unregister',
+    'parse', 'validate', 'sanitize', 'normalize',
+    'cache', 'invalidate', 'flush', 'persist', 'load',
+    'embed', 'distill', 'train', 'consolidate', 'recall',
+];
+// A trajectory step might contain code blocks, file refs, or natural prose;
+// these regexes catch the high-signal anchors without false positives on
+// general English.
+const PATH_REGEX = /\b([a-zA-Z0-9_./-]+\.(?:ts|tsx|js|jsx|mjs|cjs|py|go|rs|java|kt|swift|c|cpp|h|hpp|rb|md|json|yaml|yml|toml|sql))(?::(\d+))?\b/g;
+const SENTENCE_END = /([.!?])\s+(?=[A-Z(])/;
+function firstSentence(text) {
+    const m = text.match(SENTENCE_END);
+    if (!m || m.index === undefined)
+        return text.slice(0, SUMMARY_MAX);
+    return text.slice(0, m.index + 1).slice(0, SUMMARY_MAX);
+}
+function extractPaths(text) {
+    const found = new Set();
+    for (const m of text.matchAll(PATH_REGEX)) {
+        found.add(m[2] ? `${m[1]}:${m[2]}` : m[1]);
+    }
+    return [...found];
+}
+function extractLabels(text) {
+    const lower = text.toLowerCase();
+    const labels = new Set();
+    // Action vocabulary
+    for (const v of ACTION_VOCAB) {
+        if (lower.includes(v))
+            labels.add(v);
+    }
+    // High-signal nouns (camelCase / PascalCase identifiers, ALL_CAPS constants).
+    // Cap to the top 5 most-frequent so labels stay short.
+    const counts = new Map();
+    for (const m of text.matchAll(/\b([A-Z][a-zA-Z0-9]{3,}|[A-Z][A-Z0-9_]{3,}|[a-z][a-zA-Z0-9]{4,}[A-Z][a-zA-Z0-9]{1,})\b/g)) {
+        const t = m[1];
+        if (/^[a-zA-Z]+$/.test(t) && t.length > 20)
+            continue; // skip very long words
+        counts.set(t, (counts.get(t) ?? 0) + 1);
+    }
+    const top = [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 5);
+    for (const [t] of top)
+        labels.add(t);
+    return [...labels];
+}
+/**
+ * Distill a raw trajectory step's content into the 4-field schema.
+ * Deterministic, dependency-free, sub-millisecond.
+ */
+export function distillTrajectoryContent(raw) {
+    const text = String(raw ?? '').trim();
+    if (!text)
+        return { summary: '', detail: '', labels: [], paths: [] };
+    const paths = extractPaths(text);
+    const labels = extractLabels(text);
+    const summary = firstSentence(text);
+    // detail = everything after the first sentence, up to cap
+    const rest = text.slice(summary.length).trim();
+    const detail = rest.slice(0, DETAIL_MAX);
+    return { summary, detail, labels, paths };
+}
+/**
+ * Serialise a distilled object into a compact embedding-ready string. The
+ * resulting form is what gets embedded for retrieval — labels and paths come
+ * first so they get high attention weight in the embedding.
+ *
+ * The paper's MRR uplift comes from the ordering: high-signal tokens (labels,
+ * paths) lead, so the embedder allocates more probability mass to them.
+ */
+export function serialiseDistilled(d) {
+    const parts = [];
+    if (d.labels.length)
+        parts.push(`[${d.labels.join(', ')}]`);
+    if (d.paths.length)
+        parts.push(`paths: ${d.paths.join(' ')}`);
+    if (d.summary)
+        parts.push(d.summary);
+    if (d.detail)
+        parts.push(d.detail);
+    return parts.join('\n');
+}
+/** Convenience: distill then serialise. */
+export function distillAndSerialise(raw) {
+    return serialiseDistilled(distillTrajectoryContent(raw));
+}
+/**
+ * Compression ratio: raw bytes / distilled bytes. >1 means distilled is
+ * smaller. Used by the benchmark harness.
+ */
+export function compressionRatio(raw) {
+    const distilled = distillAndSerialise(raw);
+    if (distilled.length === 0)
+        return 1;
+    return raw.length / distilled.length;
+}
+//# sourceMappingURL=structured-distill.js.map

package/dist/src/memory/structured-distill.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"structured-distill.js","sourceRoot":"","sources":["../../../src/memory/structured-distill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAUH,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,MAAM,UAAU,GAAG,IAAI,CAAC;AAExB,gFAAgF;AAChF,0DAA0D;AAC1D,MAAM,YAAY,GAAG;IACnB,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ;IACjE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU;IAC7D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ;IACpD,UAAU,EAAE,SAAS,EAAE,SAAS;IAChC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU;IACnC,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY;IACrD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW;IAC5C,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM;IACjD,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ;CACrD,CAAC;AAEF,4EAA4E;AAC5E,yEAAyE;AACzE,mBAAmB;AACnB,MAAM,UAAU,GAAG,+HAA+H,CAAC;AACnJ,MAAM,YAAY,GAAG,sBAAsB,CAAC;AAE5C,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IACnC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IACnE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1C,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;AACpB,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,oBAAoB;IACpB,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC7B,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IACD,8EAA8E;IAC9E,uDAAuD;IACvD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,yFAAyF,CAAC,EAAE,CAAC;QACzH,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACf,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,EAAE;YAAE,SAAS,CAAC,uBAAuB;QAC7E,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1E,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG;QAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACrC,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CAAC,GAAW;IAClD,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACtC,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IAErE,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACpC,0DAA0D;IAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAEzC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC5C,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAAC,CAAmB;IACpD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5D,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC9D,IAAI,CAAC,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACnC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,2CAA2C;AAC3C,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,OAAO,kBAAkB,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,MAAM,SAAS,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAC3C,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACrC,OAAO,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;AACvC,CAAC"}