memtrails 0.0.0-stage → 0.2.0

package/dist/core/embedder.d.ts

@@ -0,0 +1,30 @@
+export declare const MEMORY_EMBEDDER: {
+    readonly id: "embeddinggemma-300m-768";
+    readonly model: "embeddinggemma-300m";
+    readonly dims: 768;
+};
+/** EmbeddingGemma's task prompts: a query and a document are embedded with different prefixes. */
+export declare const docText: (title: string, text: string) => string;
+export declare const queryText: (q: string) => string;
+export interface EmbedRoute {
+    /** The API's base, e.g. `http://localhost:8080/v1` (`/embeddings` is added). */
+    url: string;
+    apiKey?: string;
+    /** The model name the service knows it by. Default `embeddinggemma-300m`. */
+    model?: string;
+    /** Send bodies gzipped (a service that accepts `content-encoding: gzip`). */
+    gzip?: boolean;
+}
+/** The route the environment names: MEMTRAILS_EMBED_URL, MEMTRAILS_EMBED_KEY, MEMTRAILS_EMBED_MODEL; null without one. */
+export declare function resolveEmbedRoute(env?: NodeJS.ProcessEnv): EmbedRoute | null;
+/** No way to embed: neither a model directory nor a server was given. */
+export declare class EmbeddingUnavailable extends Error {
+}
+export interface EmbedderDeps {
+    route?: () => EmbedRoute | null;
+    fetchImpl?: typeof fetch;
+    /** Total operation budget, including compression, retries and all batches (default 30 s). */
+    timeoutMs?: number;
+}
+/** `embed(texts)` for memory: unit vectors in the order of `texts`, however many requests a batch needs. */
+export declare function remoteEmbedder(d?: EmbedderDeps): (texts: string[], timeoutMs?: number) => Promise<Float32Array[]>;

package/dist/core/embedder.js

@@ -0,0 +1,137 @@
+/**
+ * Embedding over HTTP, for a memory that embeds on a server: any service that speaks the OpenAI embeddings API
+ * (`POST {url}/embeddings`, `{ model, input }`) and serves EmbeddingGemma-300m — text-embeddings-inference, Infinity,
+ * Ollama (`embeddinggemma`), vLLM, your own. Vectors may come back as numbers or as little-endian float32 base64.
+ *
+ * Memory's prompts, indexes and measured thresholds are EmbeddingGemma's: a different model needs its own index
+ * (`MemoryOptions.embed` with `dims`), and its results are not the measured ones.
+ */
+import { promisify } from 'node:util';
+import { gzip } from 'node:zlib';
+export const MEMORY_EMBEDDER = { id: 'embeddinggemma-300m-768', model: 'embeddinggemma-300m', dims: 768 };
+/** EmbeddingGemma's task prompts: a query and a document are embedded with different prefixes. */
+export const docText = (title, text) => `title: ${title || 'none'} | text: ${text}`;
+export const queryText = (q) => `task: search result | query: ${q}`;
+/** The route the environment names: MEMTRAILS_EMBED_URL, MEMTRAILS_EMBED_KEY, MEMTRAILS_EMBED_MODEL; null without one. */
+export function resolveEmbedRoute(env = process.env) {
+    const url = env.MEMTRAILS_EMBED_URL?.trim();
+    if (!url)
+        return null;
+    return { url, apiKey: env.MEMTRAILS_EMBED_KEY?.trim() || undefined, model: env.MEMTRAILS_EMBED_MODEL?.trim() || undefined };
+}
+/** No way to embed: neither a model directory nor a server was given. */
+export class EmbeddingUnavailable extends Error {
+}
+const MAX_INPUTS = 256;
+const MAX_BYTES = 600_000; // JSON-escaped UTF-8 bytes in one request.
+const MAX_TEXT = 24_000;
+const IN_FLIGHT = 4;
+const compress = promisify(gzip);
+/** Validate the complete response before any vector reaches the index. Never mix dimensions or vector spaces. */
+function vectors(body, count) {
+    const data = [...(body.data ?? [])].sort((a, b) => a.index - b.index);
+    if (data.length !== count)
+        throw new Error(`embeddings: ${data.length} vectors for ${count} texts`);
+    return data.map(({ index, embedding }, n) => {
+        if (index !== n)
+            throw new Error('embeddings: missing or duplicate response index');
+        let v;
+        if (typeof embedding === 'string') {
+            const bytes = Buffer.from(embedding, 'base64');
+            if (bytes.length !== MEMORY_EMBEDDER.dims * 4 || bytes.toString('base64') !== embedding)
+                throw new Error('embeddings: invalid base64 vector');
+            v = new Float32Array(MEMORY_EMBEDDER.dims);
+            for (let i = 0; i < v.length; i++)
+                v[i] = bytes.readFloatLE(i * 4);
+        }
+        else {
+            if (!Array.isArray(embedding) || embedding.length !== MEMORY_EMBEDDER.dims || embedding.some((x) => typeof x !== 'number'))
+                throw new Error('embeddings: wrong vector dimensions');
+            v = Float32Array.from(embedding);
+        }
+        const norm = Math.hypot(...v);
+        if (!Number.isFinite(norm) || norm <= 0)
+            throw new Error('embeddings: non-finite or zero vector');
+        for (let i = 0; i < v.length; i++)
+            v[i] /= norm;
+        return v;
+    });
+}
+/** `embed(texts)` for memory: unit vectors in the order of `texts`, however many requests a batch needs. */
+export function remoteEmbedder(d = {}) {
+    const route = d.route ?? (() => resolveEmbedRoute());
+    return async (texts, timeoutMs = d.timeoutMs ?? 30_000) => {
+        if (!texts.length)
+            return [];
+        const r = route();
+        if (!r)
+            throw new EmbeddingUnavailable('no embedder: give openMemory a modelDir, an embedServer, or set MEMTRAILS_EMBED_URL');
+        const until = Date.now() + timeoutMs;
+        const remaining = () => {
+            const ms = Math.floor(until - Date.now());
+            if (ms < 1)
+                throw new Error('embeddings: operation deadline exceeded');
+            return ms;
+        };
+        const input = texts.map((t) => t.slice(0, MAX_TEXT) || ' ');
+        const parts = [];
+        let cur = [];
+        let bytes = 0;
+        input.forEach((t, i) => {
+            const size = Buffer.byteLength(JSON.stringify(t)) + 1;
+            if (cur.length && (cur.length >= MAX_INPUTS || bytes + size > MAX_BYTES)) {
+                parts.push({ at: i - cur.length, texts: cur });
+                cur = [];
+                bytes = 0;
+            }
+            cur.push(t);
+            bytes += size;
+        });
+        if (cur.length)
+            parts.push({ at: input.length - cur.length, texts: cur });
+        const call = async (part) => {
+            remaining();
+            const json = Buffer.from(JSON.stringify({ model: r.model ?? MEMORY_EMBEDDER.model, input: part.texts }));
+            const body = r.gzip ? await compress(json) : json;
+            const url = `${r.url.replace(/\/+$/, '').replace(/\/embeddings$/, '')}/embeddings`;
+            for (let attempt = 0;; attempt++) {
+                const res = await (d.fetchImpl ?? fetch)(url, {
+                    method: 'POST',
+                    signal: AbortSignal.timeout(remaining()),
+                    headers: { 'content-type': 'application/json', ...(r.gzip ? { 'content-encoding': 'gzip' } : {}), ...(r.apiKey ? { authorization: `Bearer ${r.apiKey}` } : {}) },
+                    body: new Uint8Array(body),
+                });
+                if (res.ok) {
+                    const payload = await res.json();
+                    remaining();
+                    vectors(payload, part.texts.length).forEach((v, k) => { out[part.at + k] = v; });
+                    return;
+                }
+                const message = (await res.text()).slice(0, 200); // Drain the response before retrying (keep-alive socket).
+                if ((res.status === 429 || res.status >= 500) && attempt < 3) {
+                    const delay = 500 * 2 ** attempt;
+                    if (remaining() <= delay)
+                        throw new Error('embeddings: operation deadline exceeded');
+                    await new Promise((ok) => setTimeout(ok, delay));
+                    continue;
+                }
+                throw new Error(`embeddings ${res.status}: ${message}`);
+            }
+        };
+        const out = new Array(input.length);
+        let next = 0;
+        let failed = false;
+        await Promise.all(Array.from({ length: Math.min(IN_FLIGHT, parts.length) }, async () => {
+            while (!failed && next < parts.length) {
+                try {
+                    await call(parts[next++]);
+                }
+                catch (err) {
+                    failed = true;
+                    throw err;
+                }
+            }
+        }));
+        return out;
+    };
+}

package/dist/core/episodes.d.ts

@@ -0,0 +1,32 @@
+/**
+ * How a record reads: to the index (its body, every word of which is searched in chunks) and to the consolidation model.
+ *
+ * A conversation's exchange keeps the format consolidation was measured on: what was asked, the steps, what was said.
+ * Any other record reads as itself: its title, then its text.
+ */
+/** What the index and the consolidation model read of a record (store.ts StmEntry). */
+export interface ReadableRequest {
+    uid: string;
+    /** The record's kind; `conversation` for an exchange of a conversation. */
+    engine?: string;
+    typed: 'human' | 'unknown';
+    request: string;
+    reply: string;
+    /** Everything the other side said in an exchange, whole. */
+    said: string;
+    steps: string[];
+    files: string[];
+    ts: number | null;
+}
+export declare const isConversation: (e: {
+    engine?: string;
+}) => boolean;
+/** A record's body: what its key summarises, its chunks are cut from, and opening it shows — whole, never shortened. */
+export declare function episodeBody(e: Pick<ReadableRequest, 'request' | 'reply' | 'files'> & Partial<Pick<ReadableRequest, 'said' | 'steps' | 'engine'>>): string;
+/** For each day a request was made, the days a time said relatively that day means — yesterday, the week before, the
+ *  last weekend, each of the six days before by name, this week, last month, last year (UTC as the requests' times) —
+ *  so the model reads "last week" or "on Friday" off a line instead of counting. "Last week" is the seven days before,
+ *  as people mean it, not the Monday-to-Sunday week before this one: measured on LoCoMo, the calendar week put events
+ *  said on a Sunday thirteen days back, a week early. */
+export declare function calendarOf(tss: number[]): string;
+export declare function renderForExtraction(e: ReadableRequest, n: number): string;

package/dist/core/episodes.js

@@ -0,0 +1,53 @@
+/**
+ * How a record reads: to the index (its body, every word of which is searched in chunks) and to the consolidation model.
+ *
+ * A conversation's exchange keeps the format consolidation was measured on: what was asked, the steps, what was said.
+ * Any other record reads as itself: its title, then its text.
+ */
+export const isConversation = (e) => e.engine === 'conversation';
+/** A record's body: what its key summarises, its chunks are cut from, and opening it shows — whole, never shortened. */
+export function episodeBody(e) {
+    if (e.engine !== undefined && !isConversation(e))
+        return e.reply.trim() ? `${e.request}\n${e.reply}` : e.request;
+    const parts = [`Asked: ${e.request}`];
+    if (e.steps?.length)
+        parts.push(`Steps:\n${e.steps.join('\n')}`);
+    parts.push(`Done: ${e.said?.trim() ? e.said : e.reply}`);
+    if (e.files.length)
+        parts.push(`Files: ${e.files.join(', ')}`);
+    return parts.join('\n');
+}
+/** For each day a request was made, the days a time said relatively that day means — yesterday, the week before, the
+ *  last weekend, each of the six days before by name, this week, last month, last year (UTC as the requests' times) —
+ *  so the model reads "last week" or "on Friday" off a line instead of counting. "Last week" is the seven days before,
+ *  as people mean it, not the Monday-to-Sunday week before this one: measured on LoCoMo, the calendar week put events
+ *  said on a Sunday thirteen days back, a week early. */
+export function calendarOf(tss) {
+    const DAY = 86_400_000;
+    const iso = (t) => new Date(t).toISOString().slice(0, 10);
+    const wd = (t) => new Date(t).toLocaleDateString('en-GB', { weekday: 'short', timeZone: 'UTC' });
+    const days = [...new Set(tss.filter((t) => t > 0).map((t) => Math.floor(t / DAY) * DAY))].sort((a, b) => a - b);
+    return days.map((d) => {
+        const dt = new Date(d);
+        const monday = d - ((dt.getUTCDay() + 6) % 7) * DAY;
+        const sunday = d - (dt.getUTCDay() === 0 ? 7 : dt.getUTCDay()) * DAY;
+        const lastMonth = new Date(Date.UTC(dt.getUTCFullYear(), dt.getUTCMonth() - 1, 1)).toISOString().slice(0, 7);
+        const before = [2, 3, 4, 5, 6, 7].map((n) => `${wd(d - n * DAY)} ${iso(d - n * DAY)}`).join(', ');
+        return `- ${wd(d)} ${iso(d)}: yesterday ${wd(d - DAY)} ${iso(d - DAY)} · the days before ${before} · last week (the week before) ${iso(d - 7 * DAY)} to ${iso(d - DAY)} · last weekend ${iso(sunday - DAY)} to ${iso(sunday)} · this week ${iso(monday)} to ${iso(monday + 6 * DAY)} · last month ${lastMonth} · last year ${dt.getUTCFullYear() - 1}`;
+    }).join('\n');
+}
+export function renderForExtraction(e, n) {
+    const when = e.ts ? new Date(e.ts).toISOString().slice(0, 16).replace('T', ' ') : '';
+    // Whole: a record too long for one window is read in parts and its parts joined (extract.ts), never shortened.
+    if (e.engine !== undefined && !isConversation(e))
+        return [`## Record ${n} · ${when} · ${e.engine}`, e.request, e.reply].filter(Boolean).join('\n');
+    const who = e.typed === 'human' ? 'typed by the person' : 'sent to the agent';
+    const parts = [`## Request ${n} · ${when} · ${who}`, e.request];
+    if (e.steps.length)
+        parts.push(`Steps:\n${e.steps.join('\n')}`);
+    if (e.said)
+        parts.push(`Agent said:\n${e.said}`);
+    if (e.files.length)
+        parts.push(`Files changed: ${e.files.join(', ')}`);
+    return parts.join('\n');
+}

package/dist/core/extract.d.ts

@@ -0,0 +1,142 @@
+import { type Ask } from '../llm.js';
+import { type ReadableRequest } from './episodes.js';
+import type { MemoryKind, MemoryOrigin, MemoryScope } from './store.js';
+export declare const WINDOW_CHARS = 60000;
+/** The answer's room. A window's kept episodes, items, cues and concepts measured on a whole real history: 7,056
+ *  output tokens at the median, 9,223 at p90 — and 3% of windows ran past 12,000, cut off mid-JSON. A cut answer is
+ *  no answer, and the worker asks the same (newest) window again: one long window stalled consolidation for good. */
+export declare const EXTRACT_MAX_TOKENS = 32000;
+declare const OUTCOMES: readonly ["success", "partial", "fail", "uncertain"];
+export type Outcome = (typeof OUTCOMES)[number];
+declare const PHASES: readonly ["done", "plan", "ongoing", "state"];
+export interface Candidate {
+    kind: MemoryKind;
+    title: string;
+    summary: string;
+    body: string;
+    appliesWhen: string;
+    scope: MemoryScope;
+    /** Episode uids it was learned from. */
+    provenance: string[];
+    confidence: 'verified' | 'stated';
+    importance: number;
+    /** Requests that would need it (the retrieval cues). */
+    cues: string[];
+    /** What it is about, as the conversation names it (the graph's concepts). */
+    concepts: string[];
+    /** Where it came from, when a guard checked it (ConsolidateDeps.guard). */
+    origin?: MemoryOrigin;
+}
+/** One dated thing the conversation told — kept as said, never merged with a different one (the Mem0 lesson: a list
+ *  question needs every event, and a date question the event's own date). */
+export interface Observation {
+    text: string;
+    about: string;
+    /** YYYY-MM-DD, YYYY-MM or YYYY — or empty. */
+    eventAt: string;
+    /** For what lasts: when it began and when it ended — each YYYY-MM-DD, YYYY-MM or YYYY; absent when not given. */
+    startAt?: string;
+    endAt?: string;
+    phase: (typeof PHASES)[number];
+    /** Episode uids it was told in. */
+    provenance: string[];
+    concepts: string[];
+    /** Questions it answers, in the asker's words (searched beside its text, quieter). */
+    cues: string[];
+    origin?: MemoryOrigin;
+}
+export interface KeptEpisode {
+    uid: string;
+    title: string;
+    summary: string;
+    cues: string[];
+    concepts: string[];
+    outcome?: Outcome;
+    origin?: MemoryOrigin;
+}
+/** A cue list as memory keeps it: non-empty, scrubbed, no repeats — every one the model wrote, whole. */
+export declare function cleanCues(raw: unknown): string[];
+/** A concept list as memory keeps it: named as written (scrubbed), one per concept key — every one, whole. */
+export declare function cleanConcepts(raw: unknown): string[];
+export interface Extraction {
+    summary: string;
+    candidates: Candidate[];
+    episodes: KeptEpisode[];
+    observations: Observation[];
+    dropped: number;
+}
+type RawObservation = {
+    text?: string;
+    about?: string;
+    event_date?: string;
+    start_date?: string;
+    end_date?: string;
+    phase?: string;
+    requests?: number[];
+    concepts?: unknown;
+    cues?: unknown;
+};
+/** The observations as memory keeps them: with text, from a request of this part, scrubbed; a date only when it is one. */
+export declare function cleanObservations(raw: RawObservation[], uidOf: (n: number) => string | undefined): Observation[];
+/** The windows a conversation's pending requests are read in: whole requests, at most WINDOW_CHARS — a request longer
+ *  than that alone in its window, read there in parts (extractWindow). */
+export declare function windowsOf<T extends ReadableRequest>(eps: T[], max?: number): T[][];
+type RawItem = {
+    kind?: string;
+    title?: string;
+    summary?: string;
+    body?: string;
+    applies_when?: string;
+    scope?: string;
+    requests?: number[];
+    confidence?: string;
+    importance?: number;
+    contested?: boolean;
+    ambiguous?: boolean;
+    cues?: unknown;
+    concepts?: unknown;
+};
+/** What survives the gate. Pure: the rules, in one place. `outcomeOf` is how each request ended (the
+ *  episodes' outcomes): a kind that says "this works" is dropped when every request it came from failed or is unknown. */
+export declare function gate(raw: RawItem[], uidOf: (n: number) => string | undefined, outcomeOf?: (n: number) => Outcome | undefined): {
+    kept: Candidate[];
+    dropped: number;
+};
+export type { Ask } from '../llm.js';
+/** A request's text in parts of at most `max` characters, split at line ends (a longer line at `max`) — every
+ *  character in exactly one part. */
+export declare function partsOf(text: string, max?: number): string[];
+export interface ExtractOptions {
+    ask?: Ask; /** false: observations are read apart (observeWindow). */
+    observations?: boolean;
+    /** Passages of other conversations related to this part (consolidate.ts): what a description refers to is named. */
+    known?: string[];
+}
+/** Read one window. `unavailable`/`busy` come back as they are, so the worker can pause. */
+export declare function extractWindow(win: ReadableRequest[], meta: {
+    cwd: string | null;
+    engine: string;
+    title: string;
+}, opts?: ExtractOptions): Promise<Extraction | {
+    unavailable: string;
+} | {
+    busy: true;
+}>;
+/** Observations read apart, Mem0's way: a small window of requests in one focused ask, with what came just before it
+ *  as context — what a large window's one ask for everything let slip (the events a list question needs). A request
+ *  longer than the window is read in parts, in order, each part with the one before as its context. */
+export declare function observeWindow(win: ReadableRequest[], before: ReadableRequest[], meta: {
+    cwd: string | null;
+    engine: string;
+    title: string;
+}, opts: {
+    ask?: Ask;
+    chars: number;
+    known?: string[];
+}): Promise<{
+    observations: Observation[];
+} | {
+    unavailable: string;
+} | {
+    busy: true;
+}>;