@tangleai/pipeline 0.20.0

package/CHANGELOG.md

@@ -0,0 +1,13 @@
+# @tangleai/pipeline
+
+## 0.20.0
+
+### Minor Changes
+
+- Establish the coordinated 0.20.0 release with JavaScript and TypeScript declaration distributions, preserved public subpaths and JSON schemas, and verified Node and Bun consumers. Use published JarenJS 0.83.3 fixes without a consumer installation patch. Prepare versions before release commits, verify locally, push directly to main and publish CI-verified tarballs. Deploy the website independently from local Tangle workspace source with JarenJS packages from npm, verifying dependency sources and the live commit.
+
+### Patch Changes
+
+- Updated dependencies
+  - @tangleai/core@0.20.0
+  - @tangleai/memory@0.20.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Joham (jklarenbeek@gmail.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,7 @@
+# @tangleai/pipeline
+
+Tangle AI pipeline — the memory loop as a jaren-dag document, executed by @jarenjs/flow, projected to mermaid for display
+
+Install with `npm install @tangleai/pipeline`. The npm distribution provides ESM JavaScript, TypeScript declarations, and the documented package subpaths for Node 24 and Bun 1.4 or newer.
+
+See the [Tangle documentation](https://github.com/jklarenbeek/tangleai#readme) for architecture, examples, and runtime requirements. All public Tangle packages use one coordinated version.

package/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "@tangleai/pipeline",
+  "version": "0.20.0",
+  "description": "Tangle AI pipeline — the memory loop as a jaren-dag document, executed by @jarenjs/flow, projected to mermaid for display",
+  "license": "MIT",
+  "type": "module",
+  "main": "./src/index.js",
+  "exports": {
+    ".": {
+      "types": "./src/index.d.ts",
+      "import": "./src/index.js",
+      "default": "./src/index.js"
+    },
+    "./dag": {
+      "types": "./src/dag.d.ts",
+      "import": "./src/dag.js",
+      "default": "./src/dag.js"
+    },
+    "./run": {
+      "types": "./src/run.d.ts",
+      "import": "./src/run.js",
+      "default": "./src/run.js"
+    },
+    "./mermaid": {
+      "types": "./src/mermaid.d.ts",
+      "import": "./src/mermaid.js",
+      "default": "./src/mermaid.js"
+    },
+    "./standins": {
+      "types": "./src/standins.d.ts",
+      "import": "./src/standins.js",
+      "default": "./src/standins.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "engines": {
+    "node": ">=24"
+  },
+  "sideEffects": false,
+  "dependencies": {
+    "@tangleai/core": "^0.20.0",
+    "@tangleai/memory": "^0.20.0",
+    "@jarenjs/ai": "0.83.3",
+    "@jarenjs/flow": "0.83.3",
+    "@jarenjs/json": "0.83.3",
+    "@jarenjs/mermaid": "0.83.3"
+  },
+  "private": false,
+  "types": "./src/index.d.ts",
+  "files": [
+    "src/**/*.js",
+    "src/**/*.d.ts",
+    "schemas/**/*.json",
+    "README.md",
+    "LICENSE",
+    "CHANGELOG.md"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jklarenbeek/tangleai.git",
+    "directory": "packages/pipeline"
+  }
+}

package/src/dag.d.ts

@@ -0,0 +1,59 @@
+/**
+ * The Tangle memory loop as a `jaren-dag` 0.1 document.
+ *
+ * This is the SAME loop the walking skeleton runs inline, lifted into a
+ * flow document so that (a) @jarenjs/flow executes it with real
+ * per-node observability (`onNode` → the run log → the desktop DAG
+ * page, current and historical), and (b) @jarenjs/mermaid can project
+ * the topology for display from the document itself — the picture the
+ * user sees IS the thing that runs, not an illustration of it.
+ *
+ * Stage order is the twice-learned design rule: the novelty gate only
+ * filters near-verbatim repeats, contradictions are resolved BEFORE
+ * crystallization (a contradiction is ~0.95-similar to what it
+ * contradicts), and the crystallizer merges only what survived with its
+ * meaning intact.
+ *
+ * Each stage hands its summary to the `report` output node through a
+ * ported edge; the unported spine carries the data each next stage
+ * needs. Handlers are injected at compile time (`createPipeline`) — the
+ * document names them, the host provides them, jarenjs's DI rule.
+ */
+export declare const PIPELINE_DAG: {
+    $dag: string;
+    nodes: {
+        observations: {
+            kind: string;
+        };
+        embed: {
+            kind: string;
+            run: string;
+        };
+        novelty: {
+            kind: string;
+            run: string;
+        };
+        contradiction: {
+            kind: string;
+            run: string;
+        };
+        crystallize: {
+            kind: string;
+            run: string;
+        };
+        report: {
+            kind: string;
+        };
+    };
+    edges: ({
+        from: string;
+        to: string;
+        port?: undefined;
+    } | {
+        from: string;
+        to: string;
+        port: string;
+    })[];
+};
+/** The node ids, in pipeline order — what the UI renders as the status strip. */
+export declare const PIPELINE_NODES: readonly ["observations", "embed", "novelty", "contradiction", "crystallize", "report"];

package/src/dag.js

@@ -0,0 +1,44 @@
+/**
+ * The Tangle memory loop as a `jaren-dag` 0.1 document.
+ *
+ * This is the SAME loop the walking skeleton runs inline, lifted into a
+ * flow document so that (a) @jarenjs/flow executes it with real
+ * per-node observability (`onNode` → the run log → the desktop DAG
+ * page, current and historical), and (b) @jarenjs/mermaid can project
+ * the topology for display from the document itself — the picture the
+ * user sees IS the thing that runs, not an illustration of it.
+ *
+ * Stage order is the twice-learned design rule: the novelty gate only
+ * filters near-verbatim repeats, contradictions are resolved BEFORE
+ * crystallization (a contradiction is ~0.95-similar to what it
+ * contradicts), and the crystallizer merges only what survived with its
+ * meaning intact.
+ *
+ * Each stage hands its summary to the `report` output node through a
+ * ported edge; the unported spine carries the data each next stage
+ * needs. Handlers are injected at compile time (`createPipeline`) — the
+ * document names them, the host provides them, jarenjs's DI rule.
+ */
+export const PIPELINE_DAG = {
+    $dag: '0.1',
+    nodes: {
+        observations: { kind: 'input' },
+        embed: { kind: 'task', run: 'embed' },
+        novelty: { kind: 'task', run: 'novelty' },
+        contradiction: { kind: 'task', run: 'contradiction' },
+        crystallize: { kind: 'task', run: 'crystallize' },
+        report: { kind: 'output' },
+    },
+    edges: [
+        { from: 'observations', to: 'embed' },
+        { from: 'embed', to: 'novelty' },
+        { from: 'novelty', to: 'contradiction' },
+        { from: 'contradiction', to: 'crystallize' },
+        { from: 'embed', to: 'report', port: 'embed' },
+        { from: 'novelty', to: 'report', port: 'novelty' },
+        { from: 'contradiction', to: 'report', port: 'contradiction' },
+        { from: 'crystallize', to: 'report', port: 'crystallize' },
+    ],
+};
+/** The node ids, in pipeline order — what the UI renders as the status strip. */
+export const PIPELINE_NODES = ['observations', 'embed', 'novelty', 'contradiction', 'crystallize', 'report'];

package/src/index.d.ts

@@ -0,0 +1,7 @@
+/** @tangleai/pipeline barrel. */
+export { PIPELINE_DAG, PIPELINE_NODES } from './dag.ts';
+export { createPipeline, DEFAULT_THRESHOLDS } from './run.ts';
+export type { Pipeline, PipelineOptions, PipelineReport, PipelineThresholds, DagNodeRecord, } from './run.ts';
+export { dagToMermaid } from './mermaid.ts';
+export { createOfflineEmbedder, OFFLINE_EMBEDDER_DIMS, numericContrastJudge } from './standins.ts';
+export type { Judge } from './standins.ts';

package/src/index.js

@@ -0,0 +1,5 @@
+/** @tangleai/pipeline barrel. */
+export { PIPELINE_DAG, PIPELINE_NODES } from "./dag.js";
+export { createPipeline, DEFAULT_THRESHOLDS } from "./run.js";
+export { dagToMermaid } from "./mermaid.js";
+export { createOfflineEmbedder, OFFLINE_EMBEDDER_DIMS, numericContrastJudge } from "./standins.js";

package/src/mermaid.d.ts

@@ -0,0 +1,12 @@
+/**
+ * The pipeline document, projected to Mermaid text.
+ *
+ * Uses the suite's own projection chain — the `dag-to-flowchart` JSLT
+ * stylesheet shipped by @jarenjs/mermaid, `transformJson` from
+ * @jarenjs/json, `toMermaid` over a diagram document — so the drawing
+ * is DERIVED from the executable document, never hand-drawn. Node
+ * statuses are deliberately NOT painted into the diagram; the UI
+ * renders them as a strip beside it (data changes fast, topology
+ * doesn't).
+ */
+export declare function dagToMermaid(doc?: any): string;

package/src/mermaid.js

@@ -0,0 +1,19 @@
+/**
+ * The pipeline document, projected to Mermaid text.
+ *
+ * Uses the suite's own projection chain — the `dag-to-flowchart` JSLT
+ * stylesheet shipped by @jarenjs/mermaid, `transformJson` from
+ * @jarenjs/json, `toMermaid` over a diagram document — so the drawing
+ * is DERIVED from the executable document, never hand-drawn. Node
+ * statuses are deliberately NOT painted into the diagram; the UI
+ * renders them as a strip beside it (data changes fast, topology
+ * doesn't).
+ */
+import { toMermaid, diagramDocument } from '@jarenjs/mermaid';
+import { transformJson } from '@jarenjs/json/jslt';
+import dagToFlowchart from '@jarenjs/mermaid/stylesheets/dag-to-flowchart.jslt.json' with { type: 'json' };
+import { PIPELINE_DAG } from "./dag.js";
+export function dagToMermaid(doc = PIPELINE_DAG) {
+    const ast = transformJson(dagToFlowchart, doc);
+    return toMermaid(diagramDocument('flowchart', {}, ast, { hash: '0', direction: 'TD', title: null }));
+}

package/src/run.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Compiling and running the pipeline document.
+ *
+ * `createPipeline` binds the DAG's named task handlers to injected
+ * seams — the store (any MemoryStore: in-memory for tests and the pages
+ * demo, SQLite in the desktop app), an embedder (the @jarenjs/ai seam:
+ * `{ embed, model, dims }` — `createEmbeddingClient` for a wire,
+ * `createHashEmbedder` as the offline default), a contradiction judge,
+ * a clock, and the tuned thresholds (salvaged memflow defaults). The
+ * document stays pure JSON; everything replaceable arrives here.
+ *
+ * Every vector the embed node writes carries its identity
+ * (`embeddedBy: { model, dims }`), the jarenjs rule that lets every
+ * later policy refuse to compare vectors from two models.
+ *
+ * `run()` returns a compact report; the units themselves live in the
+ * store, not in the return value. Per-node timing/status records stream
+ * through `onNode` exactly as @jarenjs/flow emits them.
+ */
+import type { Embedder } from '@jarenjs/ai/embed';
+import { type MemoryStore, type MemoryUnitInput } from '@tangleai/memory';
+import { type Judge } from './standins.ts';
+export interface PipelineThresholds {
+    /** Near-verbatim repeat filter; keep HIGH so contradictions survive. */
+    novelty?: number;
+    /** Similar-pair candidate threshold for the judge. */
+    contradiction?: number;
+    /** Paraphrase merge threshold. */
+    crystallize?: number;
+}
+export interface PipelineOptions {
+    store: MemoryStore;
+    embedder?: Embedder;
+    judge?: Judge;
+    now?: () => string;
+    thresholds?: PipelineThresholds;
+}
+export interface DagNodeRecord {
+    id: string;
+    status: 'ok' | 'error' | 'aborted' | 'restored';
+    ms: number;
+}
+export interface PipelineReport {
+    observations: number;
+    embedded: number;
+    model: string;
+    novelty: {
+        admitted: number;
+        filtered: number;
+    };
+    /** Every attempt is judged or failed; every confirmed verdict is applied or skipped. */
+    contradiction: {
+        attempted: number;
+        judged: number;
+        judgeFailures: number;
+        confirmed: number;
+        contradictions: number;
+        applicationSkips: number;
+        resolutions: string[];
+    };
+    /** Every planned merge is written or skipped. */
+    crystallize: {
+        examined: number;
+        planned: number;
+        merged: number;
+        applicationSkips: number;
+    };
+    memories: {
+        live: number;
+        total: number;
+    };
+}
+export interface Pipeline {
+    run(observations: MemoryUnitInput[], options?: {
+        signal?: AbortSignal;
+        onNode?: (record: DagNodeRecord) => void;
+    }): Promise<PipelineReport>;
+}
+export declare const DEFAULT_THRESHOLDS: Required<PipelineThresholds>;
+export declare function createPipeline(options: PipelineOptions): Pipeline;

package/src/run.js

@@ -0,0 +1,108 @@
+/**
+ * Compiling and running the pipeline document.
+ *
+ * `createPipeline` binds the DAG's named task handlers to injected
+ * seams — the store (any MemoryStore: in-memory for tests and the pages
+ * demo, SQLite in the desktop app), an embedder (the @jarenjs/ai seam:
+ * `{ embed, model, dims }` — `createEmbeddingClient` for a wire,
+ * `createHashEmbedder` as the offline default), a contradiction judge,
+ * a clock, and the tuned thresholds (salvaged memflow defaults). The
+ * document stays pure JSON; everything replaceable arrives here.
+ *
+ * Every vector the embed node writes carries its identity
+ * (`embeddedBy: { model, dims }`), the jarenjs rule that lets every
+ * later policy refuse to compare vectors from two models.
+ *
+ * `run()` returns a compact report; the units themselves live in the
+ * store, not in the return value. Per-node timing/status records stream
+ * through `onNode` exactly as @jarenjs/flow emits them.
+ */
+import { compileDag } from '@jarenjs/flow';
+import { createMemoryUnit, noveltyGate, planContradictionPairs, resolveContradictions, planCrystallization, applyCrystallization, } from '@tangleai/memory';
+import { createOfflineEmbedder, numericContrastJudge } from "./standins.js";
+import { PIPELINE_DAG } from "./dag.js";
+export const DEFAULT_THRESHOLDS = {
+    novelty: 0.97,
+    contradiction: 0.8,
+    crystallize: 0.9,
+};
+export function createPipeline(options) {
+    const store = options.store;
+    const embedder = options.embedder ?? createOfflineEmbedder();
+    const judge = options.judge ?? numericContrastJudge();
+    const now = options.now ?? (() => new Date().toISOString());
+    const thresholds = { ...DEFAULT_THRESHOLDS, ...options.thresholds };
+    const tasks = {
+        async embed({ input }, signal) {
+            const observations = input;
+            const missing = observations.filter((o) => o.embedding === undefined);
+            const vectors = missing.length > 0
+                ? await embedder.embed(missing.map((o) => o.text), { signal })
+                : [];
+            // the identity: the embedder's settled width, or the reply's
+            const dims = embedder.dims ?? vectors[0]?.length;
+            const embeddedBy = dims === undefined ? undefined : { model: embedder.model, dims };
+            let next = 0;
+            const units = observations.map((o) => {
+                if (o.embedding !== undefined)
+                    return createMemoryUnit({ ...o, confidence: o.confidence ?? 0.5 });
+                // the seam answers Float32Array; the record stores plain numbers
+                const vector = Array.from(vectors[next++]);
+                return createMemoryUnit({ ...o, embedding: vector, embeddedBy, confidence: o.confidence ?? 0.5 });
+            });
+            return { units, embedded: missing.length, model: embedder.model };
+        },
+        async novelty({ input }) {
+            const units = input.units;
+            const existing = await store.list();
+            const { novel, filtered } = noveltyGate(units, existing, { threshold: thresholds.novelty });
+            for (const unit of novel)
+                await store.put(unit);
+            return { admitted: novel.length, filtered: filtered.length };
+        },
+        async contradiction() {
+            const units = await store.list();
+            const pairs = planContradictionPairs(units, { threshold: thresholds.contradiction });
+            const outcome = await resolveContradictions(store, pairs, { now, judge });
+            return {
+                attempted: outcome.attempted,
+                judged: outcome.judged,
+                judgeFailures: outcome.judgeFailures,
+                confirmed: outcome.confirmed,
+                contradictions: outcome.contradictions,
+                applicationSkips: outcome.applicationSkips,
+                resolutions: outcome.resolutions.map((r) => r.text),
+            };
+        },
+        async crystallize() {
+            const plan = planCrystallization(await store.list(), { threshold: thresholds.crystallize });
+            const outcome = await applyCrystallization(store, plan, { now });
+            return {
+                examined: plan.examined,
+                planned: outcome.planned,
+                merged: outcome.crystallized,
+                applicationSkips: outcome.applicationSkips,
+            };
+        },
+    };
+    const dag = compileDag(PIPELINE_DAG, { tasks });
+    return {
+        async run(observations, runOptions = {}) {
+            const result = await dag.run(observations, {
+                signal: runOptions.signal,
+                onNode: runOptions.onNode,
+            });
+            const all = await store.list();
+            const live = all.filter((u) => u.supersededBy === undefined);
+            return {
+                observations: observations.length,
+                embedded: result.embed.embedded,
+                model: result.embed.model,
+                novelty: result.novelty,
+                contradiction: result.contradiction,
+                crystallize: result.crystallize,
+                memories: { live: live.length, total: all.length },
+            };
+        },
+    };
+}

package/src/standins.d.ts

@@ -0,0 +1,41 @@
+/**
+ * The two offline stand-ins — an embedder and a contradiction judge —
+ * that let the whole pipeline run with zero network, promoted out of
+ * `examples/skeleton.ts` because the desktop app and the pages demo
+ * need them as real seams.
+ *
+ * `createOfflineEmbedder` is @jarenjs/ai's `createHashEmbedder` — the
+ * suite's own deterministic hashed-trigram reference, behind the same
+ * `{ embed, model, dims }` seam a wire client fills — at the width the
+ * pipeline's thresholds were measured against. Measured over the
+ * skeleton corpus (2026-08-26, `hash-trigram-<dims>`, cosine), against
+ * the defaults novelty 0.97 / crystallize 0.9 / contradiction 0.8:
+ *
+ *   dims  repeat  paraphrase  contradiction  max-unrelated
+ *    64   0.989   0.931       0.964          0.606   ← every pair on its side of every threshold
+ *   128   0.985   0.902       0.954          0.424   ← paraphrase a hair over the 0.9 merge line
+ *   256   0.984   0.888       0.941          0.323   ← paraphrase under the merge line: never crystallized
+ *   512   0.983   0.884       0.936          0.225
+ *
+ * 64 — the suite's own default — is the width with margin on all four:
+ * wider buckets spread the shared trigrams thinner, so the paraphrase
+ * loses its merge long before an unrelated pair threatens the 0.8
+ * judge line. It stays demo-grade and lexical — a real model behind
+ * the same seam is the upgrade.
+ *
+ * `numericContrastJudge` flags two records that agree in words but
+ * disagree in figures — the rule stand-in for an LLM contradiction
+ * judge (`createStructuredOutput` over `contradictionMessages`).
+ */
+import { type Embedder } from '@jarenjs/ai/embed';
+import type { MemoryUnit } from '@tangleai/core/schemas/memory';
+import type { ContradictionVerdict } from '@tangleai/memory';
+/** The measured width (see the header). */
+export declare const OFFLINE_EMBEDDER_DIMS = 64;
+/** The offline embedder: the suite's hash reference at the measured width. */
+export declare function createOfflineEmbedder(): Embedder & {
+    dims: number;
+};
+export type Judge = (a: MemoryUnit, b: MemoryUnit) => Promise<ContradictionVerdict>;
+/** Same subject, different figures → contradiction; newer text wins. */
+export declare function numericContrastJudge(): Judge;

package/src/standins.js

@@ -0,0 +1,47 @@
+/**
+ * The two offline stand-ins — an embedder and a contradiction judge —
+ * that let the whole pipeline run with zero network, promoted out of
+ * `examples/skeleton.ts` because the desktop app and the pages demo
+ * need them as real seams.
+ *
+ * `createOfflineEmbedder` is @jarenjs/ai's `createHashEmbedder` — the
+ * suite's own deterministic hashed-trigram reference, behind the same
+ * `{ embed, model, dims }` seam a wire client fills — at the width the
+ * pipeline's thresholds were measured against. Measured over the
+ * skeleton corpus (2026-08-26, `hash-trigram-<dims>`, cosine), against
+ * the defaults novelty 0.97 / crystallize 0.9 / contradiction 0.8:
+ *
+ *   dims  repeat  paraphrase  contradiction  max-unrelated
+ *    64   0.989   0.931       0.964          0.606   ← every pair on its side of every threshold
+ *   128   0.985   0.902       0.954          0.424   ← paraphrase a hair over the 0.9 merge line
+ *   256   0.984   0.888       0.941          0.323   ← paraphrase under the merge line: never crystallized
+ *   512   0.983   0.884       0.936          0.225
+ *
+ * 64 — the suite's own default — is the width with margin on all four:
+ * wider buckets spread the shared trigrams thinner, so the paraphrase
+ * loses its merge long before an unrelated pair threatens the 0.8
+ * judge line. It stays demo-grade and lexical — a real model behind
+ * the same seam is the upgrade.
+ *
+ * `numericContrastJudge` flags two records that agree in words but
+ * disagree in figures — the rule stand-in for an LLM contradiction
+ * judge (`createStructuredOutput` over `contradictionMessages`).
+ */
+import { createHashEmbedder } from '@jarenjs/ai/embed';
+/** The measured width (see the header). */
+export const OFFLINE_EMBEDDER_DIMS = 64;
+/** The offline embedder: the suite's hash reference at the measured width. */
+export function createOfflineEmbedder() {
+    return createHashEmbedder({ dims: OFFLINE_EMBEDDER_DIMS });
+}
+/** Same subject, different figures → contradiction; newer text wins. */
+export function numericContrastJudge() {
+    const numbers = (t) => t.match(/\d+(?:\.\d+)?/g)?.join(',') ?? '';
+    return async (a, b) => {
+        if (numbers(a.text) !== numbers(b.text)) {
+            const newer = a.at <= b.at ? b : a;
+            return { contradiction: true, reason: 'same subject, different figures', resolution: newer.text };
+        }
+        return { contradiction: false };
+    };
+}