@f0rbit/corpus 0.1.0

package/dist/codec.js

@@ -0,0 +1,21 @@
+export function json_codec(schema) {
+    return {
+        content_type: "application/json",
+        encode: (value) => new TextEncoder().encode(JSON.stringify(value)),
+        decode: (bytes) => schema.parse(JSON.parse(new TextDecoder().decode(bytes))),
+    };
+}
+export function text_codec() {
+    return {
+        content_type: "text/plain",
+        encode: (value) => new TextEncoder().encode(value),
+        decode: (bytes) => new TextDecoder().decode(bytes),
+    };
+}
+export function binary_codec() {
+    return {
+        content_type: "application/octet-stream",
+        encode: (value) => value,
+        decode: (bytes) => bytes,
+    };
+}

package/dist/corpus.d.ts

@@ -0,0 +1,3 @@
+import type { CorpusBuilder } from './types';
+export declare function create_corpus(): CorpusBuilder<{}>;
+//# sourceMappingURL=corpus.d.ts.map

package/dist/corpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus.d.ts","sourceRoot":"","sources":["../corpus.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAmB,aAAa,EAA0B,MAAM,SAAS,CAAA;AAGrF,wBAAgB,aAAa,IAAI,aAAa,CAAC,EAAE,CAAC,CAoCjD"}

package/dist/corpus.js

@@ -0,0 +1,31 @@
+import { create_store } from './store';
+export function create_corpus() {
+    let backend = null;
+    const definitions = [];
+    const builder = {
+        with_backend(b) {
+            backend = b;
+            return builder;
+        },
+        with_store(definition) {
+            definitions.push(definition);
+            return builder;
+        },
+        build() {
+            if (!backend) {
+                throw new Error('Backend is required. Call with_backend() first.');
+            }
+            const b = backend;
+            const stores = {};
+            for (const def of definitions) {
+                stores[def.id] = create_store(b, def);
+            }
+            return {
+                stores,
+                metadata: b.metadata,
+                data: b.data,
+            };
+        },
+    };
+    return builder;
+}

package/dist/hash.d.ts

@@ -0,0 +1,2 @@
+export declare function compute_hash(data: Uint8Array): Promise<string>;
+//# sourceMappingURL=hash.d.ts.map

package/dist/hash.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../hash.ts"],"names":[],"mappings":"AAAA,wBAAsB,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAIpE"}

package/dist/hash.js

@@ -0,0 +1,5 @@
+export async function compute_hash(data) {
+    const hash_buffer = await crypto.subtle.digest('SHA-256', data);
+    const hash_array = new Uint8Array(hash_buffer);
+    return Array.from(hash_array).map(b => b.toString(16).padStart(2, '0')).join('');
+}

package/dist/index.d.ts

@@ -0,0 +1,13 @@
+export { create_corpus } from './corpus';
+export { create_store } from './store';
+export { create_memory_backend, type MemoryBackendOptions } from './backend/memory';
+export { create_file_backend, type FileBackendConfig } from './backend/file';
+export { create_cloudflare_backend, type CloudflareBackendConfig } from './backend/cloudflare';
+export { json_codec, text_codec, binary_codec } from './codec';
+export { corpus_snapshots, type CorpusSnapshotRow, type CorpusSnapshotInsert } from './schema';
+export { compute_hash } from './hash';
+export { generate_version } from './version';
+export type { ContentType, ParentRef, SnapshotMeta, Snapshot, DataHandle, MetadataClient, DataClient, ListOpts, Backend, Codec, Store, StoreDefinition, PutOpts, CorpusBuilder, Corpus, CorpusError, Result, CorpusEvent, EventHandler, } from './types';
+export { ok, err, define_store } from './types';
+export { createCorpusInfra, CORPUS_MIGRATION_SQL, type CorpusInfra, type CorpusInfraConfig } from './sst';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,qBAAqB,EAAE,KAAK,oBAAoB,EAAE,MAAM,kBAAkB,CAAA;AACnF,OAAO,EAAE,mBAAmB,EAAE,KAAK,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAC5E,OAAO,EAAE,yBAAyB,EAAE,KAAK,uBAAuB,EAAE,MAAM,sBAAsB,CAAA;AAE9F,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAE9D,OAAO,EAAE,gBAAgB,EAAE,KAAK,iBAAiB,EAAE,KAAK,oBAAoB,EAAE,MAAM,UAAU,CAAA;AAE9F,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAA;AACrC,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAA;AAE5C,YAAY,EACV,WAAW,EACX,SAAS,EACT,YAAY,EACZ,QAAQ,EACR,UAAU,EACV,cAAc,EACd,UAAU,EACV,QAAQ,EACR,OAAO,EACP,KAAK,EACL,KAAK,EACL,eAAe,EACf,OAAO,EACP,aAAa,EACb,MAAM,EACN,WAAW,EACX,MAAM,EACN,WAAW,EACX,YAAY,GACb,MAAM,SAAS,CAAA;AAEhB,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAE/C,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,KAAK,WAAW,EAAE,KAAK,iBAAiB,EAAE,MAAM,OAAO,CAAA"}

package/dist/index.js

@@ -0,0 +1,11 @@
+export { create_corpus } from './corpus';
+export { create_store } from './store';
+export { create_memory_backend } from './backend/memory';
+export { create_file_backend } from './backend/file';
+export { create_cloudflare_backend } from './backend/cloudflare';
+export { json_codec, text_codec, binary_codec } from './codec';
+export { corpus_snapshots } from './schema';
+export { compute_hash } from './hash';
+export { generate_version } from './version';
+export { ok, err, define_store } from './types';
+export { createCorpusInfra, CORPUS_MIGRATION_SQL } from './sst';

package/dist/schema.d.ts

@@ -0,0 +1,198 @@
+export declare const corpus_snapshots: import("drizzle-orm/sqlite-core").SQLiteTableWithColumns<{
+    name: "corpus_snapshots";
+    schema: undefined;
+    columns: {
+        store_id: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: "store_id";
+            tableName: "corpus_snapshots";
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
+        version: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: "version";
+            tableName: "corpus_snapshots";
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
+        parents: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: "parents";
+            tableName: "corpus_snapshots";
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
+        created_at: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: "created_at";
+            tableName: "corpus_snapshots";
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
+        invoked_at: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: "invoked_at";
+            tableName: "corpus_snapshots";
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
+        content_hash: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: "content_hash";
+            tableName: "corpus_snapshots";
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
+        content_type: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: "content_type";
+            tableName: "corpus_snapshots";
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
+        size_bytes: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: "size_bytes";
+            tableName: "corpus_snapshots";
+            dataType: "number";
+            columnType: "SQLiteInteger";
+            data: number;
+            driverParam: number;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {}>;
+        data_key: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: "data_key";
+            tableName: "corpus_snapshots";
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
+        tags: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: "tags";
+            tableName: "corpus_snapshots";
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
+    };
+    dialect: "sqlite";
+}>;
+export type CorpusSnapshotRow = typeof corpus_snapshots.$inferSelect;
+export type CorpusSnapshotInsert = typeof corpus_snapshots.$inferInsert;
+//# sourceMappingURL=schema.d.ts.map

package/dist/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../schema.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAgB1B,CAAA;AAEH,MAAM,MAAM,iBAAiB,GAAG,OAAO,gBAAgB,CAAC,YAAY,CAAA;AACpE,MAAM,MAAM,oBAAoB,GAAG,OAAO,gBAAgB,CAAC,YAAY,CAAA"}

package/dist/schema.js

@@ -0,0 +1,18 @@
+import { sqliteTable, text, integer, primaryKey, index } from 'drizzle-orm/sqlite-core';
+export const corpus_snapshots = sqliteTable('corpus_snapshots', {
+    store_id: text('store_id').notNull(),
+    version: text('version').notNull(),
+    parents: text('parents').notNull(),
+    created_at: text('created_at').notNull(),
+    invoked_at: text('invoked_at'),
+    content_hash: text('content_hash').notNull(),
+    content_type: text('content_type').notNull(),
+    size_bytes: integer('size_bytes').notNull(),
+    data_key: text('data_key').notNull(),
+    tags: text('tags'),
+}, (table) => ({
+    pk: primaryKey({ columns: [table.store_id, table.version] }),
+    created_idx: index('idx_store_created').on(table.store_id, table.created_at),
+    hash_idx: index('idx_content_hash').on(table.store_id, table.content_hash),
+    data_key_idx: index('idx_data_key').on(table.data_key),
+}));

package/dist/sst.d.ts

@@ -0,0 +1,18 @@
+export type CorpusInfraConfig = {
+    name: string;
+    bucket_name?: string;
+    database_name?: string;
+};
+export type CorpusInfra = {
+    database: {
+        name: string;
+    };
+    bucket: {
+        name: string;
+    };
+    database_name: string;
+    bucket_name: string;
+};
+export declare function createCorpusInfra(name: string, config?: Partial<CorpusInfraConfig>): CorpusInfra;
+export declare const CORPUS_MIGRATION_SQL = "\nCREATE TABLE IF NOT EXISTS corpus_snapshots (\n  store_id TEXT NOT NULL,\n  version TEXT NOT NULL,\n  parents TEXT NOT NULL,\n  created_at TEXT NOT NULL,\n  invoked_at TEXT,\n  content_hash TEXT NOT NULL,\n  content_type TEXT NOT NULL,\n  size_bytes INTEGER NOT NULL,\n  data_key TEXT NOT NULL,\n  tags TEXT,\n  PRIMARY KEY (store_id, version)\n);\n\nCREATE INDEX IF NOT EXISTS idx_store_created ON corpus_snapshots(store_id, created_at);\nCREATE INDEX IF NOT EXISTS idx_content_hash ON corpus_snapshots(store_id, content_hash);\nCREATE INDEX IF NOT EXISTS idx_data_key ON corpus_snapshots(data_key);\n";
+//# sourceMappingURL=sst.d.ts.map

package/dist/sst.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sst.d.ts","sourceRoot":"","sources":["../sst.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,aAAa,CAAC,EAAE,MAAM,CAAA;CACvB,CAAA;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1B,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;IACxB,aAAa,EAAE,MAAM,CAAA;IACrB,WAAW,EAAE,MAAM,CAAA;CACpB,CAAA;AAED,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAClC,WAAW,CAUb;AAED,eAAO,MAAM,oBAAoB,imBAkBhC,CAAA"}

package/dist/sst.js

@@ -0,0 +1,29 @@
+export function createCorpusInfra(name, config) {
+    const database_name = config?.database_name ?? `${name}Db`;
+    const bucket_name = config?.bucket_name ?? `${name}Bucket`;
+    return {
+        database: { name: database_name },
+        bucket: { name: bucket_name },
+        database_name,
+        bucket_name,
+    };
+}
+export const CORPUS_MIGRATION_SQL = `
+CREATE TABLE IF NOT EXISTS corpus_snapshots (
+  store_id TEXT NOT NULL,
+  version TEXT NOT NULL,
+  parents TEXT NOT NULL,
+  created_at TEXT NOT NULL,
+  invoked_at TEXT,
+  content_hash TEXT NOT NULL,
+  content_type TEXT NOT NULL,
+  size_bytes INTEGER NOT NULL,
+  data_key TEXT NOT NULL,
+  tags TEXT,
+  PRIMARY KEY (store_id, version)
+);
+
+CREATE INDEX IF NOT EXISTS idx_store_created ON corpus_snapshots(store_id, created_at);
+CREATE INDEX IF NOT EXISTS idx_content_hash ON corpus_snapshots(store_id, content_hash);
+CREATE INDEX IF NOT EXISTS idx_data_key ON corpus_snapshots(data_key);
+`;

package/dist/store.d.ts

@@ -0,0 +1,3 @@
+import type { Backend, Store, StoreDefinition } from './types';
+export declare function create_store<T>(backend: Backend, definition: StoreDefinition<string, T>): Store<T>;
+//# sourceMappingURL=store.d.ts.map

package/dist/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,eAAe,EAA8C,MAAM,SAAS,CAAA;AAK1G,wBAAgB,YAAY,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CA+IlG"}

package/dist/store.js

@@ -0,0 +1,125 @@
+import { ok, err } from './types';
+import { compute_hash } from './hash';
+import { generate_version } from './version';
+export function create_store(backend, definition) {
+    const { id, codec } = definition;
+    function emit(event) {
+        backend.on_event?.(event);
+    }
+    function make_data_key(store_id, content_hash) {
+        return `${store_id}/${content_hash}`;
+    }
+    return {
+        id,
+        codec,
+        async put(data, opts) {
+            const version = generate_version();
+            let bytes;
+            try {
+                bytes = codec.encode(data);
+            }
+            catch (cause) {
+                const error = { kind: 'encode_error', cause: cause };
+                emit({ type: 'error', error });
+                return err(error);
+            }
+            const content_hash = await compute_hash(bytes);
+            // deduplication: reuse existing data_key if content already exists
+            const existing = await backend.metadata.find_by_hash(id, content_hash);
+            const deduplicated = existing !== null;
+            const data_key = deduplicated ? existing.data_key : make_data_key(id, content_hash);
+            if (!deduplicated) {
+                const data_result = await backend.data.put(data_key, bytes);
+                if (!data_result.ok) {
+                    emit({ type: 'error', error: data_result.error });
+                    return data_result;
+                }
+            }
+            emit({ type: 'data_put', store_id: id, version, size_bytes: bytes.length, deduplicated });
+            const meta = {
+                store_id: id,
+                version,
+                parents: opts?.parents ?? [],
+                created_at: new Date(),
+                invoked_at: opts?.invoked_at,
+                content_hash,
+                content_type: codec.content_type,
+                size_bytes: bytes.length,
+                data_key,
+                tags: opts?.tags,
+            };
+            const meta_result = await backend.metadata.put(meta);
+            if (!meta_result.ok) {
+                emit({ type: 'error', error: meta_result.error });
+                return meta_result;
+            }
+            emit({ type: 'snapshot_put', store_id: id, version, content_hash, deduplicated });
+            return ok(meta);
+        },
+        async get(version) {
+            const meta_result = await backend.metadata.get(id, version);
+            if (!meta_result.ok) {
+                emit({ type: 'snapshot_get', store_id: id, version, found: false });
+                return meta_result;
+            }
+            const meta = meta_result.value;
+            const data_result = await backend.data.get(meta.data_key);
+            if (!data_result.ok) {
+                emit({ type: 'error', error: data_result.error });
+                return data_result;
+            }
+            const bytes = await data_result.value.bytes();
+            let data;
+            try {
+                data = codec.decode(bytes);
+            }
+            catch (cause) {
+                const error = { kind: 'decode_error', cause: cause };
+                emit({ type: 'error', error });
+                return err(error);
+            }
+            emit({ type: 'snapshot_get', store_id: id, version, found: true });
+            return ok({ meta, data });
+        },
+        async get_latest() {
+            const meta_result = await backend.metadata.get_latest(id);
+            if (!meta_result.ok) {
+                return meta_result;
+            }
+            const meta = meta_result.value;
+            const data_result = await backend.data.get(meta.data_key);
+            if (!data_result.ok) {
+                return data_result;
+            }
+            const bytes = await data_result.value.bytes();
+            let data;
+            try {
+                data = codec.decode(bytes);
+            }
+            catch (cause) {
+                const error = { kind: 'decode_error', cause: cause };
+                emit({ type: 'error', error });
+                return err(error);
+            }
+            return ok({ meta, data });
+        },
+        async get_meta(version) {
+            return backend.metadata.get(id, version);
+        },
+        list(opts) {
+            return backend.metadata.list(id, opts);
+        },
+        async delete(version) {
+            const meta_result = await backend.metadata.get(id, version);
+            if (!meta_result.ok) {
+                return meta_result;
+            }
+            const delete_meta_result = await backend.metadata.delete(id, version);
+            if (!delete_meta_result.ok) {
+                return delete_meta_result;
+            }
+            emit({ type: 'meta_delete', store_id: id, version });
+            return ok(undefined);
+        },
+    };
+}