@f0rbit/corpus 0.1.5 → 0.1.6

package/dist/backend/cloudflare.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cloudflare.d.ts","sourceRoot":"","sources":["../../backend/cloudflare.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,KAAK,EAAE,OAAO,EAAwF,YAAY,EAAE,MAAM,UAAU,CAAC;AAI5I,KAAK,UAAU,GAAG;IAAE,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAA;CAAE,CAAC;AACxD,KAAK,QAAQ,GAAG;IACf,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QAAC,WAAW,EAAE,MAAM,OAAO,CAAC,WAAW,CAAC,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IACpH,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACnF,MAAM,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;CACvD,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACrC,EAAE,EAAE,UAAU,CAAC;IACf,EAAE,EAAE,QAAQ,CAAC;IACb,QAAQ,CAAC,EAAE,YAAY,CAAC;CACxB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,uBAAuB,GAAG,OAAO,CA8OlF"}
+{"version":3,"file":"cloudflare.d.ts","sourceRoot":"","sources":["../../backend/cloudflare.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,KAAK,EAAE,OAAO,EAAiE,YAAY,EAAE,MAAM,UAAU,CAAC;AAMrH,KAAK,UAAU,GAAG;IAAE,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAA;CAAE,CAAC;AACxD,KAAK,QAAQ,GAAG;IACf,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QAAC,WAAW,EAAE,MAAM,OAAO,CAAC,WAAW,CAAC,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IACpH,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACnF,MAAM,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;CACvD,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACrC,EAAE,EAAE,UAAU,CAAC;IACf,EAAE,EAAE,QAAQ,CAAC;IACb,QAAQ,CAAC,EAAE,YAAY,CAAC;CACxB,CAAC;AA6IF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,uBAAuB,GAAG,OAAO,CAmOlF"}

package/dist/backend/cloudflare.js

@@ -2,10 +2,138 @@
  * @module Backends
  * @description Cloudflare Workers storage backend using D1 and R2.
  */
-import { eq, and, desc, lt, gt, like, sql } from "drizzle-orm";
+import { eq, and, desc, lt, gt, like, sql, inArray } from "drizzle-orm";
 import { drizzle } from "drizzle-orm/d1";
+import { create_emitter, parse_snapshot_meta } from "../utils";
 import { ok, err } from "../types";
 import { corpus_snapshots } from "../schema";
+import { corpus_observations, create_observations_client } from "../observations";
+function create_cloudflare_storage(db) {
+    return {
+        async put_row(row) {
+            try {
+                await db.insert(corpus_observations).values(row);
+                return ok(row);
+            }
+            catch (cause) {
+                return err({
+                    kind: "storage_error",
+                    cause: cause,
+                    operation: "observations.put"
+                });
+            }
+        },
+        async get_row(id) {
+            try {
+                const rows = await db
+                    .select()
+                    .from(corpus_observations)
+                    .where(eq(corpus_observations.id, id))
+                    .limit(1);
+                return ok(rows[0] ?? null);
+            }
+            catch (cause) {
+                return err({
+                    kind: "storage_error",
+                    cause: cause,
+                    operation: "observations.get"
+                });
+            }
+        },
+        async *query_rows(opts = {}) {
+            const conditions = [];
+            if (opts.type) {
+                if (Array.isArray(opts.type)) {
+                    conditions.push(inArray(corpus_observations.type, opts.type));
+                }
+                else {
+                    conditions.push(eq(corpus_observations.type, opts.type));
+                }
+            }
+            if (opts.source_store_id) {
+                conditions.push(eq(corpus_observations.source_store_id, opts.source_store_id));
+            }
+            if (opts.source_version) {
+                conditions.push(eq(corpus_observations.source_version, opts.source_version));
+            }
+            if (opts.source_prefix) {
+                conditions.push(like(corpus_observations.source_version, `${opts.source_prefix}%`));
+            }
+            if (opts.created_after) {
+                conditions.push(gt(corpus_observations.created_at, opts.created_after));
+            }
+            if (opts.created_before) {
+                conditions.push(lt(corpus_observations.created_at, opts.created_before));
+            }
+            if (opts.observed_after) {
+                conditions.push(gt(corpus_observations.observed_at, opts.observed_after));
+            }
+            if (opts.observed_before) {
+                conditions.push(lt(corpus_observations.observed_at, opts.observed_before));
+            }
+            let query = db
+                .select()
+                .from(corpus_observations)
+                .where(conditions.length > 0 ? and(...conditions) : undefined)
+                .orderBy(desc(corpus_observations.created_at));
+            if (opts.limit) {
+                query = query.limit(opts.limit);
+            }
+            const rows = await query;
+            for (const row of rows) {
+                yield row;
+            }
+        },
+        async delete_row(id) {
+            try {
+                const existing = await db
+                    .select()
+                    .from(corpus_observations)
+                    .where(eq(corpus_observations.id, id))
+                    .limit(1);
+                if (existing.length === 0) {
+                    return ok(false);
+                }
+                await db.delete(corpus_observations).where(eq(corpus_observations.id, id));
+                return ok(true);
+            }
+            catch (cause) {
+                return err({
+                    kind: "storage_error",
+                    cause: cause,
+                    operation: "observations.delete"
+                });
+            }
+        },
+        async delete_by_source(store_id, version, path) {
+            try {
+                const conditions = [
+                    eq(corpus_observations.source_store_id, store_id),
+                    eq(corpus_observations.source_version, version)
+                ];
+                if (path !== undefined) {
+                    conditions.push(eq(corpus_observations.source_path, path));
+                }
+                const toDelete = await db
+                    .select()
+                    .from(corpus_observations)
+                    .where(and(...conditions));
+                const count = toDelete.length;
+                if (count > 0) {
+                    await db.delete(corpus_observations).where(and(...conditions));
+                }
+                return ok(count);
+            }
+            catch (cause) {
+                return err({
+                    kind: "storage_error",
+                    cause: cause,
+                    operation: "observations.delete_by_source"
+                });
+            }
+        }
+    };
+}
 /**
  * Creates a Cloudflare Workers storage backend using D1 and R2.
  * @category Backends
@@ -45,22 +173,9 @@ import { corpus_snapshots } from "../schema";
 export function create_cloudflare_backend(config) {
     const db = drizzle(config.d1);
     const { r2, on_event } = config;
-    function emit(event) {
-        on_event?.(event);
-    }
-    function row_to_meta(row) {
-        return {
-            store_id: row.store_id,
-            version: row.version,
-            parents: JSON.parse(row.parents),
-            created_at: new Date(row.created_at),
-            invoked_at: row.invoked_at ? new Date(row.invoked_at) : undefined,
-            content_hash: row.content_hash,
-            content_type: row.content_type,
-            size_bytes: row.size_bytes,
-            data_key: row.data_key,
-            tags: row.tags ? JSON.parse(row.tags) : undefined,
-        };
+    const emit = create_emitter(on_event);
+    function snapshot_row_to_meta(row) {
+        return parse_snapshot_meta(row);
     }
     const metadata = {
         async get(store_id, version) {
@@ -75,7 +190,7 @@ export function create_cloudflare_backend(config) {
                 if (!row) {
                     return err({ kind: "not_found", store_id, version });
                 }
-                return ok(row_to_meta(row));
+                return ok(snapshot_row_to_meta(row));
             }
             catch (cause) {
                 const error = { kind: "storage_error", cause: cause, operation: "metadata.get" };
@@ -152,7 +267,7 @@ export function create_cloudflare_backend(config) {
             const rows = await query;
             let count = 0;
             for (const row of rows) {
-                const meta = row_to_meta(row);
+                const meta = snapshot_row_to_meta(row);
                 if (opts?.tags?.length && !opts.tags.every(t => meta.tags?.includes(t))) {
                     continue;
                 }
@@ -168,7 +283,7 @@ export function create_cloudflare_backend(config) {
                 if (!row) {
                     return err({ kind: "not_found", store_id, version: "latest" });
                 }
-                return ok(row_to_meta(row));
+                return ok(snapshot_row_to_meta(row));
             }
             catch (cause) {
                 const error = { kind: "storage_error", cause: cause, operation: "metadata.get_latest" };
@@ -186,7 +301,7 @@ export function create_cloudflare_backend(config) {
               AND json_extract(value, '$.version') = ${parent_version}
           )`);
             for (const row of rows) {
-                yield row_to_meta(row);
+                yield snapshot_row_to_meta(row);
             }
         },
         async find_by_hash(store_id, content_hash) {
@@ -197,7 +312,7 @@ export function create_cloudflare_backend(config) {
                     .where(and(eq(corpus_snapshots.store_id, store_id), eq(corpus_snapshots.content_hash, content_hash)))
                     .limit(1);
                 const row = rows[0];
-                return row ? row_to_meta(row) : null;
+                return row ? snapshot_row_to_meta(row) : null;
             }
             catch {
                 return null;
@@ -255,5 +370,7 @@ export function create_cloudflare_backend(config) {
             }
         },
     };
-    return { metadata, data, on_event };
+    const storage = create_cloudflare_storage(db);
+    const observations = create_observations_client(storage, metadata);
+    return { metadata, data, observations, on_event };
 }

package/dist/backend/file.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"file.d.ts","sourceRoot":"","sources":["../../backend/file.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAwF,YAAY,EAAE,MAAM,UAAU,CAAC;AAK5I,MAAM,MAAM,iBAAiB,GAAG;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,YAAY,CAAC;CACxB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAkMtE"}
+{"version":3,"file":"file.d.ts","sourceRoot":"","sources":["../../backend/file.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAiE,YAAY,EAAE,MAAM,UAAU,CAAC;AAQrH,MAAM,MAAM,iBAAiB,GAAG;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,YAAY,CAAC;CACxB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CA6MtE"}

package/dist/backend/file.js

@@ -2,7 +2,9 @@
  * @module Backends
  * @description File-system storage backend for local persistence.
  */
+import { create_observations_client, create_observations_storage } from "../observations";
 import { ok, err } from "../types";
+import { to_bytes, create_emitter, filter_snapshots, parse_snapshot_meta } from "../utils";
 import { mkdir, readdir } from "node:fs/promises";
 import { join, dirname } from "node:path";
 /**
@@ -38,9 +40,7 @@ import { join, dirname } from "node:path";
  */
 export function create_file_backend(config) {
     const { base_path, on_event } = config;
-    function emit(event) {
-        on_event?.(event);
-    }
+    const emit = create_emitter(on_event);
     function meta_path(store_id) {
         return join(base_path, store_id, "_meta.json");
     }
@@ -54,13 +54,8 @@ export function create_file_backend(config) {
             return new Map();
         try {
             const content = await file.text();
-            const entries = JSON.parse(content, (key, value) => {
-                if (key === "created_at" || key === "invoked_at") {
-                    return value ? new Date(value) : value;
-                }
-                return value;
-            });
-            return new Map(entries);
+            const entries = JSON.parse(content);
+            return new Map(entries.map(([key, raw]) => [key, parse_snapshot_meta(raw)]));
         }
         catch {
             return new Map();
@@ -98,20 +93,9 @@ export function create_file_backend(config) {
         },
         async *list(store_id, opts) {
             const store_meta = await read_store_meta(store_id);
-            const matches = Array.from(store_meta.values())
-                .filter(meta => {
-                if (opts?.before && meta.created_at >= opts.before)
-                    return false;
-                if (opts?.after && meta.created_at <= opts.after)
-                    return false;
-                if (opts?.tags?.length && !opts.tags.every(t => meta.tags?.includes(t)))
-                    return false;
-                return true;
-            })
-                .sort((a, b) => b.created_at.getTime() - a.created_at.getTime());
-            const limit = opts?.limit ?? Infinity;
+            const filtered = filter_snapshots(Array.from(store_meta.values()), opts);
             let count = 0;
-            for (const meta of matches.slice(0, limit)) {
+            for (const meta of filtered) {
                 yield meta;
                 count++;
             }
@@ -174,21 +158,8 @@ export function create_file_backend(config) {
             const path = data_path(data_key);
             await mkdir(dirname(path), { recursive: true });
             try {
-                if (input instanceof Uint8Array) {
-                    await Bun.write(path, input);
-                }
-                else {
-                    const chunks = [];
-                    const reader = input.getReader();
-                    while (true) {
-                        const { done, value } = await reader.read();
-                        if (done)
-                            break;
-                        chunks.push(value);
-                    }
-                    const bytes = concat_bytes(chunks);
-                    await Bun.write(path, bytes);
-                }
+                const bytes = await to_bytes(input);
+                await Bun.write(path, bytes);
                 return ok(undefined);
             }
             catch (cause) {
@@ -214,15 +185,43 @@ export function create_file_backend(config) {
             return file.exists();
         },
     };
-    return { metadata, data, on_event };
-}
-function concat_bytes(chunks) {
-    const total = chunks.reduce((sum, c) => sum + c.length, 0);
-    const result = new Uint8Array(total);
-    let offset = 0;
-    for (const chunk of chunks) {
-        result.set(chunk, offset);
-        offset += chunk.length;
+    const file_path = join(base_path, "_observations.json");
+    async function read_observations() {
+        const file = Bun.file(file_path);
+        if (!(await file.exists()))
+            return [];
+        try {
+            return await file.json();
+        }
+        catch {
+            return [];
+        }
     }
-    return result;
+    async function write_observations(rows) {
+        await Bun.write(file_path, JSON.stringify(rows, null, 2));
+    }
+    const storage = create_observations_storage({
+        get_all: read_observations,
+        set_all: write_observations,
+        get_one: async (id) => {
+            const rows = await read_observations();
+            return rows.find(r => r.id === id) ?? null;
+        },
+        add_one: async (row) => {
+            const rows = await read_observations();
+            rows.push(row);
+            await write_observations(rows);
+        },
+        remove_one: async (id) => {
+            const rows = await read_observations();
+            const idx = rows.findIndex(r => r.id === id);
+            if (idx === -1)
+                return false;
+            rows.splice(idx, 1);
+            await write_observations(rows);
+            return true;
+        }
+    });
+    const observations = create_observations_client(storage, metadata);
+    return { metadata, data, observations, on_event };
 }

package/dist/backend/layered.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"layered.d.ts","sourceRoot":"","sources":["../../backend/layered.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,OAAO,EAA6E,MAAM,UAAU,CAAA;AAGlH,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,OAAO,EAAE,CAAA;IACf,KAAK,EAAE,OAAO,EAAE,CAAA;IAChB,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,CAAA;CAClC,CAAA;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CA4I9E"}
+{"version":3,"file":"layered.d.ts","sourceRoot":"","sources":["../../backend/layered.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAiG,MAAM,UAAU,CAAA;AAItI,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,OAAO,EAAE,CAAA;IACf,KAAK,EAAE,OAAO,EAAE,CAAA;IAChB,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,CAAA;CAClC,CAAA;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAkJ9E"}

package/dist/backend/layered.js

@@ -3,6 +3,7 @@
  * @description Layered backend for caching and replication strategies.
  */
 import { ok, err } from '../types';
+import { to_bytes } from '../utils';
 /**
  * Creates a layered backend that combines multiple backends with read/write separation.
  * @category Backends
@@ -169,25 +170,72 @@ export function create_layered_backend(options) {
             return false;
         },
     };
-    return { metadata, data };
+    const observations = createLayeredObservationsClient(read, write);
+    return {
+        metadata,
+        data,
+        ...(observations ? { observations } : {}),
+    };
 }
-async function to_bytes(data) {
-    if (data instanceof Uint8Array)
-        return data;
-    const chunks = [];
-    const reader = data.getReader();
-    while (true) {
-        const { done, value } = await reader.read();
-        if (done)
-            break;
-        chunks.push(value);
-    }
-    const total = chunks.reduce((sum, c) => sum + c.length, 0);
-    const result = new Uint8Array(total);
-    let offset = 0;
-    for (const chunk of chunks) {
-        result.set(chunk, offset);
-        offset += chunk.length;
+function createLayeredObservationsClient(readLayers, writeLayers) {
+    const readLayer = readLayers.find(l => l.observations);
+    const writeLayersWithObs = writeLayers.filter(l => l.observations);
+    if (!readLayer?.observations && writeLayersWithObs.length === 0) {
+        return undefined;
     }
-    return result;
+    const primary = readLayer?.observations;
+    return {
+        async put(type, opts) {
+            if (writeLayersWithObs.length === 0) {
+                return err({ kind: 'invalid_config', message: 'No write layers support observations' });
+            }
+            let result;
+            for (const layer of writeLayersWithObs) {
+                result = await layer.observations.put(type, opts);
+                if (!result.ok)
+                    return result;
+            }
+            return result;
+        },
+        async get(id) {
+            if (!primary) {
+                return err({ kind: 'observation_not_found', id });
+            }
+            return primary.get(id);
+        },
+        async *query(opts) {
+            if (!primary)
+                return;
+            yield* primary.query(opts);
+        },
+        async *query_meta(opts) {
+            if (!primary)
+                return;
+            yield* primary.query_meta(opts);
+        },
+        async delete(id) {
+            if (writeLayersWithObs.length === 0) {
+                return err({ kind: 'observation_not_found', id });
+            }
+            let result;
+            for (const layer of writeLayersWithObs) {
+                result = await layer.observations.delete(id);
+            }
+            return result;
+        },
+        async delete_by_source(source) {
+            let total = 0;
+            for (const layer of writeLayersWithObs) {
+                const result = await layer.observations.delete_by_source(source);
+                if (result.ok)
+                    total += result.value;
+            }
+            return ok(total);
+        },
+        async is_stale(pointer) {
+            if (!primary)
+                return false;
+            return primary.is_stale(pointer);
+        },
+    };
 }

package/dist/backend/memory.d.ts

@@ -2,7 +2,8 @@
  * @module Backends
  * @description In-memory storage backend for testing and development.
  */
-import type { Backend, EventHandler } from "../types";
+import type { Backend } from "../types";
+import type { EventHandler } from "../types";
 export type MemoryBackendOptions = {
     on_event?: EventHandler;
 };

package/dist/backend/memory.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../backend/memory.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAwF,YAAY,EAAE,MAAM,UAAU,CAAC;AAG5I,MAAM,MAAM,oBAAoB,GAAG;IAClC,QAAQ,CAAC,EAAE,YAAY,CAAC;CACxB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAgJ7E"}
+{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../backend/memory.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAiE,MAAM,UAAU,CAAC;AAKvG,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C,MAAM,MAAM,oBAAoB,GAAG;IAClC,QAAQ,CAAC,EAAE,YAAY,CAAC;CACxB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CA4I7E"}

package/dist/backend/memory.js

@@ -2,7 +2,9 @@
  * @module Backends
  * @description In-memory storage backend for testing and development.
  */
+import { create_observations_client, create_observations_storage } from "../observations";
 import { ok, err } from "../types";
+import { to_bytes, create_emitter, filter_snapshots } from "../utils";
 /**
  * Creates an in-memory storage backend.
  * @category Backends
@@ -32,10 +34,9 @@ import { ok, err } from "../types";
 export function create_memory_backend(options) {
     const meta_store = new Map();
     const data_store = new Map();
+    const observation_store = new Map();
     const on_event = options?.on_event;
-    function emit(event) {
-        on_event?.(event);
-    }
+    const emit = create_emitter(on_event);
     function make_meta_key(store_id, version) {
         return `${store_id}:${version}`;
     }
@@ -60,23 +61,16 @@ export function create_memory_backend(options) {
         },
         async *list(store_id, opts) {
             const prefix = `${store_id}:`;
-            const matches = [];
+            const store_metas = [];
             for (const [key, meta] of meta_store) {
-                if (!key.startsWith(prefix))
-                    continue;
-                if (opts?.before && meta.created_at >= opts.before)
-                    continue;
-                if (opts?.after && meta.created_at <= opts.after)
-                    continue;
-                if (opts?.tags?.length && !opts.tags.every(t => meta.tags?.includes(t)))
-                    continue;
-                matches.push(meta);
+                if (key.startsWith(prefix)) {
+                    store_metas.push(meta);
+                }
             }
-            matches.sort((a, b) => b.created_at.getTime() - a.created_at.getTime());
-            const limit = opts?.limit ?? Infinity;
+            const filtered = filter_snapshots(store_metas, opts);
             let count = 0;
-            for (const match of matches.slice(0, limit)) {
-                yield match;
+            for (const meta of filtered) {
+                yield meta;
                 count++;
             }
             emit({ type: "meta_list", store_id, count });
@@ -131,21 +125,7 @@ export function create_memory_backend(options) {
             });
         },
         async put(data_key, input) {
-            let bytes;
-            if (input instanceof Uint8Array) {
-                bytes = input;
-            }
-            else {
-                const chunks = [];
-                const reader = input.getReader();
-                while (true) {
-                    const { done, value } = await reader.read();
-                    if (done)
-                        break;
-                    chunks.push(value);
-                }
-                bytes = concat_bytes(chunks);
-            }
+            const bytes = await to_bytes(input);
             data_store.set(data_key, bytes);
             return ok(undefined);
         },
@@ -157,15 +137,21 @@ export function create_memory_backend(options) {
             return data_store.has(data_key);
         },
     };
-    return { metadata, data, on_event };
-}
-function concat_bytes(chunks) {
-    const total = chunks.reduce((sum, c) => sum + c.length, 0);
-    const result = new Uint8Array(total);
-    let offset = 0;
-    for (const chunk of chunks) {
-        result.set(chunk, offset);
-        offset += chunk.length;
-    }
-    return result;
+    const storage = create_observations_storage({
+        get_all: async () => Array.from(observation_store.values()),
+        set_all: async (rows) => {
+            observation_store.clear();
+            for (const row of rows)
+                observation_store.set(row.id, row);
+        },
+        get_one: async (id) => observation_store.get(id) ?? null,
+        add_one: async (row) => { observation_store.set(row.id, row); },
+        remove_one: async (id) => {
+            const had = observation_store.has(id);
+            observation_store.delete(id);
+            return had;
+        }
+    });
+    const observations = create_observations_client(storage, metadata);
+    return { metadata, data, observations, on_event };
 }

package/dist/corpus.d.ts

@@ -64,6 +64,17 @@ export declare function create_store<T>(backend: Backend, definition: StoreDefin
  * // Type-safe access to stores
  * await corpus.stores.users.put({ name: 'Alice', email: 'alice@example.com' })
  * await corpus.stores.notes.put('Hello, world!')
+ *
+ * // With observations
+ * const corpus_with_obs = create_corpus()
+ *   .with_backend(create_memory_backend())
+ *   .with_store(users)
+ *   .with_observations([EntityType, SentimentType])
+ *   .build()
+ *
+ * // Pointer utilities
+ * const pointer = corpus_with_obs.create_pointer('users', 'v123', '$.name')
+ * const value = await corpus_with_obs.resolve_pointer(pointer)
  * ```
  */
 export declare function create_corpus(): CorpusBuilder<{}>;

package/dist/corpus.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"corpus.d.ts","sourceRoot":"","sources":["../corpus.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAU,aAAa,EAAE,eAAe,EAAE,KAAK,EAAqD,MAAM,SAAS,CAAA;AAIxI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAmJlG;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,aAAa,IAAI,aAAa,CAAC,EAAE,CAAC,CAoCjD"}
+{"version":3,"file":"corpus.d.ts","sourceRoot":"","sources":["../corpus.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAU,aAAa,EAAE,eAAe,EAAE,KAAK,EAAyE,MAAM,SAAS,CAAA;AAM5J;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAmJlG;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,wBAAgB,aAAa,IAAI,aAAa,CAAC,EAAE,CAAC,CAiFjD"}