@f0rbit/corpus 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -0
- package/dist/backend/cloudflare.d.ts +23 -0
- package/dist/backend/cloudflare.d.ts.map +1 -0
- package/dist/backend/cloudflare.js +226 -0
- package/dist/backend/file.d.ts +7 -0
- package/dist/backend/file.d.ts.map +1 -0
- package/dist/backend/file.js +194 -0
- package/dist/backend/memory.d.ts +6 -0
- package/dist/backend/memory.d.ts.map +1 -0
- package/dist/backend/memory.js +141 -0
- package/dist/codec.d.ts +6 -0
- package/dist/codec.d.ts.map +1 -0
- package/dist/codec.js +21 -0
- package/dist/corpus.d.ts +3 -0
- package/dist/corpus.d.ts.map +1 -0
- package/dist/corpus.js +31 -0
- package/dist/hash.d.ts +2 -0
- package/dist/hash.d.ts.map +1 -0
- package/dist/hash.js +5 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/schema.d.ts +198 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +18 -0
- package/dist/sst.d.ts +18 -0
- package/dist/sst.d.ts.map +1 -0
- package/dist/sst.js +29 -0
- package/dist/store.d.ts +3 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +125 -0
- package/dist/types.d.ts +173 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +5 -0
- package/dist/version.d.ts +7 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +31 -0
- package/package.json +62 -0
package/README.md
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# corpus
|
|
2
|
+
|
|
3
|
+
a functional snapshotting library for typescript. store versioned data with lineage tracking, content deduplication, and multiple backend support (memory, file, cloudflare d1/r2).
|
|
4
|
+
|
|
5
|
+
## install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
bun add corpus
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## usage
|
|
12
|
+
|
|
13
|
+
```typescript
|
|
14
|
+
import { z } from 'zod'
|
|
15
|
+
import { create_corpus, create_memory_backend, define_store, json_codec } from 'corpus'
|
|
16
|
+
|
|
17
|
+
const TimelineSchema = z.object({
|
|
18
|
+
items: z.array(z.object({ id: z.string(), text: z.string() })),
|
|
19
|
+
})
|
|
20
|
+
|
|
21
|
+
const corpus = create_corpus()
|
|
22
|
+
.with_backend(create_memory_backend())
|
|
23
|
+
.with_store(define_store('timelines', json_codec(TimelineSchema)))
|
|
24
|
+
.build()
|
|
25
|
+
|
|
26
|
+
// typed store access - version is auto-generated
|
|
27
|
+
const result = await corpus.stores.timelines.put({
|
|
28
|
+
items: [{ id: '1', text: 'hello' }]
|
|
29
|
+
})
|
|
30
|
+
|
|
31
|
+
if (result.ok) {
|
|
32
|
+
console.log('saved:', result.value.content_hash)
|
|
33
|
+
}
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## todo
|
|
37
|
+
|
|
38
|
+
- [ ] add gzip compression codec wrapper for large json blobs
|
|
39
|
+
- [ ] add encryption codec wrapper for sensitive data at rest
|
|
40
|
+
- [ ] implement ttl/expiration support with auto-cleanup
|
|
41
|
+
- [ ] add batch operations (put_many, get_many) for bulk imports
|
|
42
|
+
- [ ] create drizzle migration files for d1 schema setup
|
|
43
|
+
- [ ] add diff(v1, v2) function for comparing json snapshots
|
|
44
|
+
- [ ] implement data compaction (merge old versions)
|
|
45
|
+
- [ ] add rate limiting awareness for cloudflare api limits
|
|
46
|
+
- [ ] create test utilities module with helpers (create_test_corpus, seed_test_data)
|
|
47
|
+
- [ ] add signed url support for direct r2 access to large files
|
|
48
|
+
- [ ] implement garbage collection for orphaned data blobs
|
|
49
|
+
- [ ] add retry logic with exponential backoff for network failures
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { Backend, EventHandler } from '../types';
|
|
2
|
+
type D1Database = {
|
|
3
|
+
prepare: (sql: string) => unknown;
|
|
4
|
+
};
|
|
5
|
+
type R2Bucket = {
|
|
6
|
+
get: (key: string) => Promise<{
|
|
7
|
+
body: ReadableStream<Uint8Array>;
|
|
8
|
+
arrayBuffer: () => Promise<ArrayBuffer>;
|
|
9
|
+
} | null>;
|
|
10
|
+
put: (key: string, data: ReadableStream<Uint8Array> | Uint8Array) => Promise<void>;
|
|
11
|
+
delete: (key: string) => Promise<void>;
|
|
12
|
+
head: (key: string) => Promise<{
|
|
13
|
+
key: string;
|
|
14
|
+
} | null>;
|
|
15
|
+
};
|
|
16
|
+
export type CloudflareBackendConfig = {
|
|
17
|
+
d1: D1Database;
|
|
18
|
+
r2: R2Bucket;
|
|
19
|
+
on_event?: EventHandler;
|
|
20
|
+
};
|
|
21
|
+
export declare function create_cloudflare_backend(config: CloudflareBackendConfig): Backend;
|
|
22
|
+
export {};
|
|
23
|
+
//# sourceMappingURL=cloudflare.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cloudflare.d.ts","sourceRoot":"","sources":["../../backend/cloudflare.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAwF,YAAY,EAAE,MAAM,UAAU,CAAA;AAI3I,KAAK,UAAU,GAAG;IAAE,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAA;CAAE,CAAA;AACvD,KAAK,QAAQ,GAAG;IACd,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QAAC,WAAW,EAAE,MAAM,OAAO,CAAC,WAAW,CAAC,CAAA;KAAE,GAAG,IAAI,CAAC,CAAA;IACnH,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAClF,MAAM,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IACtC,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAA;CACvD,CAAA;AAED,MAAM,MAAM,uBAAuB,GAAG;IACpC,EAAE,EAAE,UAAU,CAAA;IACd,EAAE,EAAE,QAAQ,CAAA;IACZ,QAAQ,CAAC,EAAE,YAAY,CAAA;CACxB,CAAA;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,uBAAuB,GAAG,OAAO,CA8PlF"}
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
import { eq, and, desc, lt, gt, like, sql } from 'drizzle-orm';
|
|
2
|
+
import { drizzle } from 'drizzle-orm/d1';
|
|
3
|
+
import { ok, err } from '../types';
|
|
4
|
+
import { corpus_snapshots } from '../schema';
|
|
5
|
+
export function create_cloudflare_backend(config) {
|
|
6
|
+
const db = drizzle(config.d1);
|
|
7
|
+
const { r2, on_event } = config;
|
|
8
|
+
function emit(event) {
|
|
9
|
+
on_event?.(event);
|
|
10
|
+
}
|
|
11
|
+
function row_to_meta(row) {
|
|
12
|
+
return {
|
|
13
|
+
store_id: row.store_id,
|
|
14
|
+
version: row.version,
|
|
15
|
+
parents: JSON.parse(row.parents),
|
|
16
|
+
created_at: new Date(row.created_at),
|
|
17
|
+
invoked_at: row.invoked_at ? new Date(row.invoked_at) : undefined,
|
|
18
|
+
content_hash: row.content_hash,
|
|
19
|
+
content_type: row.content_type,
|
|
20
|
+
size_bytes: row.size_bytes,
|
|
21
|
+
data_key: row.data_key,
|
|
22
|
+
tags: row.tags ? JSON.parse(row.tags) : undefined,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
const metadata = {
|
|
26
|
+
async get(store_id, version) {
|
|
27
|
+
try {
|
|
28
|
+
const rows = await db
|
|
29
|
+
.select()
|
|
30
|
+
.from(corpus_snapshots)
|
|
31
|
+
.where(and(eq(corpus_snapshots.store_id, store_id), eq(corpus_snapshots.version, version)))
|
|
32
|
+
.limit(1);
|
|
33
|
+
const row = rows[0];
|
|
34
|
+
emit({ type: 'meta_get', store_id, version, found: !!row });
|
|
35
|
+
if (!row) {
|
|
36
|
+
return err({ kind: 'not_found', store_id, version });
|
|
37
|
+
}
|
|
38
|
+
return ok(row_to_meta(row));
|
|
39
|
+
}
|
|
40
|
+
catch (cause) {
|
|
41
|
+
const error = { kind: 'storage_error', cause: cause, operation: 'metadata.get' };
|
|
42
|
+
emit({ type: 'error', error });
|
|
43
|
+
return err(error);
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
async put(meta) {
|
|
47
|
+
try {
|
|
48
|
+
await db
|
|
49
|
+
.insert(corpus_snapshots)
|
|
50
|
+
.values({
|
|
51
|
+
store_id: meta.store_id,
|
|
52
|
+
version: meta.version,
|
|
53
|
+
parents: JSON.stringify(meta.parents),
|
|
54
|
+
created_at: meta.created_at.toISOString(),
|
|
55
|
+
invoked_at: meta.invoked_at?.toISOString() ?? null,
|
|
56
|
+
content_hash: meta.content_hash,
|
|
57
|
+
content_type: meta.content_type,
|
|
58
|
+
size_bytes: meta.size_bytes,
|
|
59
|
+
data_key: meta.data_key,
|
|
60
|
+
tags: meta.tags ? JSON.stringify(meta.tags) : null,
|
|
61
|
+
})
|
|
62
|
+
.onConflictDoUpdate({
|
|
63
|
+
target: [corpus_snapshots.store_id, corpus_snapshots.version],
|
|
64
|
+
set: {
|
|
65
|
+
parents: JSON.stringify(meta.parents),
|
|
66
|
+
created_at: meta.created_at.toISOString(),
|
|
67
|
+
invoked_at: meta.invoked_at?.toISOString() ?? null,
|
|
68
|
+
content_hash: meta.content_hash,
|
|
69
|
+
content_type: meta.content_type,
|
|
70
|
+
size_bytes: meta.size_bytes,
|
|
71
|
+
data_key: meta.data_key,
|
|
72
|
+
tags: meta.tags ? JSON.stringify(meta.tags) : null,
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
emit({ type: 'meta_put', store_id: meta.store_id, version: meta.version });
|
|
76
|
+
return ok(undefined);
|
|
77
|
+
}
|
|
78
|
+
catch (cause) {
|
|
79
|
+
const error = { kind: 'storage_error', cause: cause, operation: 'metadata.put' };
|
|
80
|
+
emit({ type: 'error', error });
|
|
81
|
+
return err(error);
|
|
82
|
+
}
|
|
83
|
+
},
|
|
84
|
+
async delete(store_id, version) {
|
|
85
|
+
try {
|
|
86
|
+
await db
|
|
87
|
+
.delete(corpus_snapshots)
|
|
88
|
+
.where(and(eq(corpus_snapshots.store_id, store_id), eq(corpus_snapshots.version, version)));
|
|
89
|
+
emit({ type: 'meta_delete', store_id, version });
|
|
90
|
+
return ok(undefined);
|
|
91
|
+
}
|
|
92
|
+
catch (cause) {
|
|
93
|
+
const error = { kind: 'storage_error', cause: cause, operation: 'metadata.delete' };
|
|
94
|
+
emit({ type: 'error', error });
|
|
95
|
+
return err(error);
|
|
96
|
+
}
|
|
97
|
+
},
|
|
98
|
+
async *list(store_id, opts) {
|
|
99
|
+
const conditions = [like(corpus_snapshots.store_id, `${store_id}%`)];
|
|
100
|
+
if (opts?.before) {
|
|
101
|
+
conditions.push(lt(corpus_snapshots.created_at, opts.before.toISOString()));
|
|
102
|
+
}
|
|
103
|
+
if (opts?.after) {
|
|
104
|
+
conditions.push(gt(corpus_snapshots.created_at, opts.after.toISOString()));
|
|
105
|
+
}
|
|
106
|
+
let query = db
|
|
107
|
+
.select()
|
|
108
|
+
.from(corpus_snapshots)
|
|
109
|
+
.where(and(...conditions))
|
|
110
|
+
.orderBy(desc(corpus_snapshots.created_at));
|
|
111
|
+
if (opts?.limit) {
|
|
112
|
+
query = query.limit(opts.limit);
|
|
113
|
+
}
|
|
114
|
+
const rows = await query;
|
|
115
|
+
let count = 0;
|
|
116
|
+
for (const row of rows) {
|
|
117
|
+
const meta = row_to_meta(row);
|
|
118
|
+
if (opts?.tags?.length && !opts.tags.some(t => meta.tags?.includes(t))) {
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
yield meta;
|
|
122
|
+
count++;
|
|
123
|
+
}
|
|
124
|
+
emit({ type: 'meta_list', store_id, count });
|
|
125
|
+
},
|
|
126
|
+
async get_latest(store_id) {
|
|
127
|
+
try {
|
|
128
|
+
const rows = await db
|
|
129
|
+
.select()
|
|
130
|
+
.from(corpus_snapshots)
|
|
131
|
+
.where(eq(corpus_snapshots.store_id, store_id))
|
|
132
|
+
.orderBy(desc(corpus_snapshots.created_at))
|
|
133
|
+
.limit(1);
|
|
134
|
+
const row = rows[0];
|
|
135
|
+
if (!row) {
|
|
136
|
+
return err({ kind: 'not_found', store_id, version: 'latest' });
|
|
137
|
+
}
|
|
138
|
+
return ok(row_to_meta(row));
|
|
139
|
+
}
|
|
140
|
+
catch (cause) {
|
|
141
|
+
const error = { kind: 'storage_error', cause: cause, operation: 'metadata.get_latest' };
|
|
142
|
+
emit({ type: 'error', error });
|
|
143
|
+
return err(error);
|
|
144
|
+
}
|
|
145
|
+
},
|
|
146
|
+
async *get_children(parent_store_id, parent_version) {
|
|
147
|
+
const rows = await db
|
|
148
|
+
.select()
|
|
149
|
+
.from(corpus_snapshots)
|
|
150
|
+
.where(sql `EXISTS (
|
|
151
|
+
SELECT 1 FROM json_each(${corpus_snapshots.parents})
|
|
152
|
+
WHERE json_extract(value, '$.store_id') = ${parent_store_id}
|
|
153
|
+
AND json_extract(value, '$.version') = ${parent_version}
|
|
154
|
+
)`);
|
|
155
|
+
for (const row of rows) {
|
|
156
|
+
yield row_to_meta(row);
|
|
157
|
+
}
|
|
158
|
+
},
|
|
159
|
+
async find_by_hash(store_id, content_hash) {
|
|
160
|
+
try {
|
|
161
|
+
const rows = await db
|
|
162
|
+
.select()
|
|
163
|
+
.from(corpus_snapshots)
|
|
164
|
+
.where(and(eq(corpus_snapshots.store_id, store_id), eq(corpus_snapshots.content_hash, content_hash)))
|
|
165
|
+
.limit(1);
|
|
166
|
+
const row = rows[0];
|
|
167
|
+
return row ? row_to_meta(row) : null;
|
|
168
|
+
}
|
|
169
|
+
catch {
|
|
170
|
+
return null;
|
|
171
|
+
}
|
|
172
|
+
},
|
|
173
|
+
};
|
|
174
|
+
const data = {
|
|
175
|
+
async get(data_key) {
|
|
176
|
+
try {
|
|
177
|
+
const object = await r2.get(data_key);
|
|
178
|
+
emit({ type: 'data_get', store_id: data_key.split('/')[0] ?? data_key, version: data_key, found: !!object });
|
|
179
|
+
if (!object) {
|
|
180
|
+
return err({ kind: 'not_found', store_id: data_key, version: '' });
|
|
181
|
+
}
|
|
182
|
+
return ok({
|
|
183
|
+
stream: () => object.body,
|
|
184
|
+
bytes: async () => new Uint8Array(await object.arrayBuffer()),
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
catch (cause) {
|
|
188
|
+
const error = { kind: 'storage_error', cause: cause, operation: 'data.get' };
|
|
189
|
+
emit({ type: 'error', error });
|
|
190
|
+
return err(error);
|
|
191
|
+
}
|
|
192
|
+
},
|
|
193
|
+
async put(data_key, input) {
|
|
194
|
+
try {
|
|
195
|
+
await r2.put(data_key, input);
|
|
196
|
+
return ok(undefined);
|
|
197
|
+
}
|
|
198
|
+
catch (cause) {
|
|
199
|
+
const error = { kind: 'storage_error', cause: cause, operation: 'data.put' };
|
|
200
|
+
emit({ type: 'error', error });
|
|
201
|
+
return err(error);
|
|
202
|
+
}
|
|
203
|
+
},
|
|
204
|
+
async delete(data_key) {
|
|
205
|
+
try {
|
|
206
|
+
await r2.delete(data_key);
|
|
207
|
+
return ok(undefined);
|
|
208
|
+
}
|
|
209
|
+
catch (cause) {
|
|
210
|
+
const error = { kind: 'storage_error', cause: cause, operation: 'data.delete' };
|
|
211
|
+
emit({ type: 'error', error });
|
|
212
|
+
return err(error);
|
|
213
|
+
}
|
|
214
|
+
},
|
|
215
|
+
async exists(data_key) {
|
|
216
|
+
try {
|
|
217
|
+
const head = await r2.head(data_key);
|
|
218
|
+
return head !== null;
|
|
219
|
+
}
|
|
220
|
+
catch {
|
|
221
|
+
return false;
|
|
222
|
+
}
|
|
223
|
+
},
|
|
224
|
+
};
|
|
225
|
+
return { metadata, data, on_event };
|
|
226
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file.d.ts","sourceRoot":"","sources":["../../backend/file.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAwF,YAAY,EAAE,MAAM,UAAU,CAAA;AAK3I,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,CAAC,EAAE,YAAY,CAAA;CACxB,CAAA;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAqMtE"}
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { ok, err } from '../types';
|
|
2
|
+
import { mkdir, unlink, readdir } from 'node:fs/promises';
|
|
3
|
+
import { join, dirname } from 'node:path';
|
|
4
|
+
export function create_file_backend(config) {
|
|
5
|
+
const { base_path, on_event } = config;
|
|
6
|
+
function emit(event) {
|
|
7
|
+
on_event?.(event);
|
|
8
|
+
}
|
|
9
|
+
function meta_path(store_id) {
|
|
10
|
+
return join(base_path, store_id, '_meta.json');
|
|
11
|
+
}
|
|
12
|
+
function data_path(data_key) {
|
|
13
|
+
return join(base_path, '_data', `${data_key.replace(/\//g, '_')}.bin`);
|
|
14
|
+
}
|
|
15
|
+
async function read_store_meta(store_id) {
|
|
16
|
+
const path = meta_path(store_id);
|
|
17
|
+
const file = Bun.file(path);
|
|
18
|
+
if (!await file.exists())
|
|
19
|
+
return new Map();
|
|
20
|
+
try {
|
|
21
|
+
const content = await file.text();
|
|
22
|
+
const entries = JSON.parse(content, (key, value) => {
|
|
23
|
+
if (key === 'created_at' || key === 'invoked_at') {
|
|
24
|
+
return value ? new Date(value) : value;
|
|
25
|
+
}
|
|
26
|
+
return value;
|
|
27
|
+
});
|
|
28
|
+
return new Map(entries);
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return new Map();
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
async function write_store_meta(store_id, meta_map) {
|
|
35
|
+
const path = meta_path(store_id);
|
|
36
|
+
await mkdir(dirname(path), { recursive: true });
|
|
37
|
+
const entries = Array.from(meta_map.entries());
|
|
38
|
+
await Bun.write(path, JSON.stringify(entries));
|
|
39
|
+
}
|
|
40
|
+
const metadata = {
|
|
41
|
+
async get(store_id, version) {
|
|
42
|
+
const store_meta = await read_store_meta(store_id);
|
|
43
|
+
const meta = store_meta.get(version);
|
|
44
|
+
emit({ type: 'meta_get', store_id, version, found: !!meta });
|
|
45
|
+
if (!meta) {
|
|
46
|
+
return err({ kind: 'not_found', store_id, version });
|
|
47
|
+
}
|
|
48
|
+
return ok(meta);
|
|
49
|
+
},
|
|
50
|
+
async put(meta) {
|
|
51
|
+
const store_meta = await read_store_meta(meta.store_id);
|
|
52
|
+
store_meta.set(meta.version, meta);
|
|
53
|
+
await write_store_meta(meta.store_id, store_meta);
|
|
54
|
+
emit({ type: 'meta_put', store_id: meta.store_id, version: meta.version });
|
|
55
|
+
return ok(undefined);
|
|
56
|
+
},
|
|
57
|
+
async delete(store_id, version) {
|
|
58
|
+
const store_meta = await read_store_meta(store_id);
|
|
59
|
+
store_meta.delete(version);
|
|
60
|
+
await write_store_meta(store_id, store_meta);
|
|
61
|
+
emit({ type: 'meta_delete', store_id, version });
|
|
62
|
+
return ok(undefined);
|
|
63
|
+
},
|
|
64
|
+
async *list(store_id, opts) {
|
|
65
|
+
const store_meta = await read_store_meta(store_id);
|
|
66
|
+
const matches = Array.from(store_meta.values())
|
|
67
|
+
.filter(meta => {
|
|
68
|
+
if (opts?.before && meta.created_at >= opts.before)
|
|
69
|
+
return false;
|
|
70
|
+
if (opts?.after && meta.created_at <= opts.after)
|
|
71
|
+
return false;
|
|
72
|
+
if (opts?.tags?.length && !opts.tags.some(t => meta.tags?.includes(t)))
|
|
73
|
+
return false;
|
|
74
|
+
return true;
|
|
75
|
+
})
|
|
76
|
+
.sort((a, b) => b.created_at.getTime() - a.created_at.getTime());
|
|
77
|
+
const limit = opts?.limit ?? Infinity;
|
|
78
|
+
let count = 0;
|
|
79
|
+
for (const meta of matches.slice(0, limit)) {
|
|
80
|
+
yield meta;
|
|
81
|
+
count++;
|
|
82
|
+
}
|
|
83
|
+
emit({ type: 'meta_list', store_id, count });
|
|
84
|
+
},
|
|
85
|
+
async get_latest(store_id) {
|
|
86
|
+
const store_meta = await read_store_meta(store_id);
|
|
87
|
+
let latest = null;
|
|
88
|
+
for (const meta of store_meta.values()) {
|
|
89
|
+
if (!latest || meta.created_at > latest.created_at) {
|
|
90
|
+
latest = meta;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
if (!latest) {
|
|
94
|
+
return err({ kind: 'not_found', store_id, version: 'latest' });
|
|
95
|
+
}
|
|
96
|
+
return ok(latest);
|
|
97
|
+
},
|
|
98
|
+
async *get_children(parent_store_id, parent_version) {
|
|
99
|
+
try {
|
|
100
|
+
const entries = await readdir(base_path, { withFileTypes: true });
|
|
101
|
+
for (const entry of entries) {
|
|
102
|
+
if (!entry.isDirectory() || entry.name.startsWith('_'))
|
|
103
|
+
continue;
|
|
104
|
+
const store_meta = await read_store_meta(entry.name);
|
|
105
|
+
for (const meta of store_meta.values()) {
|
|
106
|
+
const is_child = meta.parents.some(p => p.store_id === parent_store_id && p.version === parent_version);
|
|
107
|
+
if (is_child)
|
|
108
|
+
yield meta;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
}
|
|
114
|
+
},
|
|
115
|
+
async find_by_hash(store_id, content_hash) {
|
|
116
|
+
const store_meta = await read_store_meta(store_id);
|
|
117
|
+
for (const meta of store_meta.values()) {
|
|
118
|
+
if (meta.content_hash === content_hash) {
|
|
119
|
+
return meta;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return null;
|
|
123
|
+
},
|
|
124
|
+
};
|
|
125
|
+
const data = {
|
|
126
|
+
async get(data_key) {
|
|
127
|
+
const path = data_path(data_key);
|
|
128
|
+
const file = Bun.file(path);
|
|
129
|
+
const found = await file.exists();
|
|
130
|
+
emit({ type: 'data_get', store_id: data_key.split('/')[0] ?? data_key, version: data_key, found });
|
|
131
|
+
if (!found) {
|
|
132
|
+
return err({ kind: 'not_found', store_id: data_key, version: '' });
|
|
133
|
+
}
|
|
134
|
+
return ok({
|
|
135
|
+
stream: () => file.stream(),
|
|
136
|
+
bytes: async () => new Uint8Array(await file.arrayBuffer()),
|
|
137
|
+
});
|
|
138
|
+
},
|
|
139
|
+
async put(data_key, input) {
|
|
140
|
+
const path = data_path(data_key);
|
|
141
|
+
await mkdir(dirname(path), { recursive: true });
|
|
142
|
+
try {
|
|
143
|
+
if (input instanceof Uint8Array) {
|
|
144
|
+
await Bun.write(path, input);
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
const chunks = [];
|
|
148
|
+
const reader = input.getReader();
|
|
149
|
+
while (true) {
|
|
150
|
+
const { done, value } = await reader.read();
|
|
151
|
+
if (done)
|
|
152
|
+
break;
|
|
153
|
+
chunks.push(value);
|
|
154
|
+
}
|
|
155
|
+
const bytes = concat_bytes(chunks);
|
|
156
|
+
await Bun.write(path, bytes);
|
|
157
|
+
}
|
|
158
|
+
return ok(undefined);
|
|
159
|
+
}
|
|
160
|
+
catch (cause) {
|
|
161
|
+
return err({ kind: 'storage_error', cause: cause, operation: 'put' });
|
|
162
|
+
}
|
|
163
|
+
},
|
|
164
|
+
async delete(data_key) {
|
|
165
|
+
const path = data_path(data_key);
|
|
166
|
+
try {
|
|
167
|
+
const file = Bun.file(path);
|
|
168
|
+
if (await file.exists()) {
|
|
169
|
+
await unlink(path);
|
|
170
|
+
}
|
|
171
|
+
return ok(undefined);
|
|
172
|
+
}
|
|
173
|
+
catch (cause) {
|
|
174
|
+
return err({ kind: 'storage_error', cause: cause, operation: 'delete' });
|
|
175
|
+
}
|
|
176
|
+
},
|
|
177
|
+
async exists(data_key) {
|
|
178
|
+
const path = data_path(data_key);
|
|
179
|
+
const file = Bun.file(path);
|
|
180
|
+
return file.exists();
|
|
181
|
+
},
|
|
182
|
+
};
|
|
183
|
+
return { metadata, data, on_event };
|
|
184
|
+
}
|
|
185
|
+
function concat_bytes(chunks) {
|
|
186
|
+
const total = chunks.reduce((sum, c) => sum + c.length, 0);
|
|
187
|
+
const result = new Uint8Array(total);
|
|
188
|
+
let offset = 0;
|
|
189
|
+
for (const chunk of chunks) {
|
|
190
|
+
result.set(chunk, offset);
|
|
191
|
+
offset += chunk.length;
|
|
192
|
+
}
|
|
193
|
+
return result;
|
|
194
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../backend/memory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAwF,YAAY,EAAE,MAAM,UAAU,CAAA;AAG3I,MAAM,MAAM,oBAAoB,GAAG;IACjC,QAAQ,CAAC,EAAE,YAAY,CAAA;CACxB,CAAA;AAED,wBAAgB,qBAAqB,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAiJ7E"}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { ok, err } from '../types';
|
|
2
|
+
export function create_memory_backend(options) {
|
|
3
|
+
const meta_store = new Map();
|
|
4
|
+
const data_store = new Map();
|
|
5
|
+
const on_event = options?.on_event;
|
|
6
|
+
function emit(event) {
|
|
7
|
+
on_event?.(event);
|
|
8
|
+
}
|
|
9
|
+
function make_meta_key(store_id, version) {
|
|
10
|
+
return `${store_id}:${version}`;
|
|
11
|
+
}
|
|
12
|
+
const metadata = {
|
|
13
|
+
async get(store_id, version) {
|
|
14
|
+
const meta = meta_store.get(make_meta_key(store_id, version));
|
|
15
|
+
emit({ type: 'meta_get', store_id, version, found: !!meta });
|
|
16
|
+
if (!meta) {
|
|
17
|
+
return err({ kind: 'not_found', store_id, version });
|
|
18
|
+
}
|
|
19
|
+
return ok(meta);
|
|
20
|
+
},
|
|
21
|
+
async put(meta) {
|
|
22
|
+
meta_store.set(make_meta_key(meta.store_id, meta.version), meta);
|
|
23
|
+
emit({ type: 'meta_put', store_id: meta.store_id, version: meta.version });
|
|
24
|
+
return ok(undefined);
|
|
25
|
+
},
|
|
26
|
+
async delete(store_id, version) {
|
|
27
|
+
meta_store.delete(make_meta_key(store_id, version));
|
|
28
|
+
emit({ type: 'meta_delete', store_id, version });
|
|
29
|
+
return ok(undefined);
|
|
30
|
+
},
|
|
31
|
+
async *list(store_id, opts) {
|
|
32
|
+
const prefix = `${store_id}:`;
|
|
33
|
+
const matches = [];
|
|
34
|
+
for (const [key, meta] of meta_store) {
|
|
35
|
+
if (!key.startsWith(prefix))
|
|
36
|
+
continue;
|
|
37
|
+
if (opts?.before && meta.created_at >= opts.before)
|
|
38
|
+
continue;
|
|
39
|
+
if (opts?.after && meta.created_at <= opts.after)
|
|
40
|
+
continue;
|
|
41
|
+
if (opts?.tags?.length && !opts.tags.some(t => meta.tags?.includes(t)))
|
|
42
|
+
continue;
|
|
43
|
+
matches.push(meta);
|
|
44
|
+
}
|
|
45
|
+
matches.sort((a, b) => b.created_at.getTime() - a.created_at.getTime());
|
|
46
|
+
const limit = opts?.limit ?? Infinity;
|
|
47
|
+
let count = 0;
|
|
48
|
+
for (const match of matches.slice(0, limit)) {
|
|
49
|
+
yield match;
|
|
50
|
+
count++;
|
|
51
|
+
}
|
|
52
|
+
emit({ type: 'meta_list', store_id, count });
|
|
53
|
+
},
|
|
54
|
+
async get_latest(store_id) {
|
|
55
|
+
let latest = null;
|
|
56
|
+
const prefix = `${store_id}:`;
|
|
57
|
+
for (const [key, meta] of meta_store) {
|
|
58
|
+
if (!key.startsWith(prefix))
|
|
59
|
+
continue;
|
|
60
|
+
if (!latest || meta.created_at > latest.created_at) {
|
|
61
|
+
latest = meta;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (!latest) {
|
|
65
|
+
return err({ kind: 'not_found', store_id, version: 'latest' });
|
|
66
|
+
}
|
|
67
|
+
return ok(latest);
|
|
68
|
+
},
|
|
69
|
+
async *get_children(parent_store_id, parent_version) {
|
|
70
|
+
for (const meta of meta_store.values()) {
|
|
71
|
+
const is_child = meta.parents.some(p => p.store_id === parent_store_id && p.version === parent_version);
|
|
72
|
+
if (is_child)
|
|
73
|
+
yield meta;
|
|
74
|
+
}
|
|
75
|
+
},
|
|
76
|
+
async find_by_hash(store_id, content_hash) {
|
|
77
|
+
const prefix = `${store_id}:`;
|
|
78
|
+
for (const [key, meta] of meta_store) {
|
|
79
|
+
if (key.startsWith(prefix) && meta.content_hash === content_hash) {
|
|
80
|
+
return meta;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return null;
|
|
84
|
+
},
|
|
85
|
+
};
|
|
86
|
+
const data = {
|
|
87
|
+
async get(data_key) {
|
|
88
|
+
const bytes = data_store.get(data_key);
|
|
89
|
+
emit({ type: 'data_get', store_id: data_key.split('/')[0] ?? data_key, version: data_key, found: !!bytes });
|
|
90
|
+
if (!bytes) {
|
|
91
|
+
return err({ kind: 'not_found', store_id: data_key, version: '' });
|
|
92
|
+
}
|
|
93
|
+
return ok({
|
|
94
|
+
stream: () => new ReadableStream({
|
|
95
|
+
start(controller) {
|
|
96
|
+
controller.enqueue(bytes);
|
|
97
|
+
controller.close();
|
|
98
|
+
}
|
|
99
|
+
}),
|
|
100
|
+
bytes: async () => bytes,
|
|
101
|
+
});
|
|
102
|
+
},
|
|
103
|
+
async put(data_key, input) {
|
|
104
|
+
let bytes;
|
|
105
|
+
if (input instanceof Uint8Array) {
|
|
106
|
+
bytes = input;
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
const chunks = [];
|
|
110
|
+
const reader = input.getReader();
|
|
111
|
+
while (true) {
|
|
112
|
+
const { done, value } = await reader.read();
|
|
113
|
+
if (done)
|
|
114
|
+
break;
|
|
115
|
+
chunks.push(value);
|
|
116
|
+
}
|
|
117
|
+
bytes = concat_bytes(chunks);
|
|
118
|
+
}
|
|
119
|
+
data_store.set(data_key, bytes);
|
|
120
|
+
return ok(undefined);
|
|
121
|
+
},
|
|
122
|
+
async delete(data_key) {
|
|
123
|
+
data_store.delete(data_key);
|
|
124
|
+
return ok(undefined);
|
|
125
|
+
},
|
|
126
|
+
async exists(data_key) {
|
|
127
|
+
return data_store.has(data_key);
|
|
128
|
+
},
|
|
129
|
+
};
|
|
130
|
+
return { metadata, data, on_event };
|
|
131
|
+
}
|
|
132
|
+
function concat_bytes(chunks) {
|
|
133
|
+
const total = chunks.reduce((sum, c) => sum + c.length, 0);
|
|
134
|
+
const result = new Uint8Array(total);
|
|
135
|
+
let offset = 0;
|
|
136
|
+
for (const chunk of chunks) {
|
|
137
|
+
result.set(chunk, offset);
|
|
138
|
+
offset += chunk.length;
|
|
139
|
+
}
|
|
140
|
+
return result;
|
|
141
|
+
}
|
package/dist/codec.d.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { ZodSchema } from "zod";
|
|
2
|
+
import type { Codec } from "./types";
|
|
3
|
+
export declare function json_codec<T>(schema: ZodSchema<T>): Codec<T>;
|
|
4
|
+
export declare function text_codec(): Codec<string>;
|
|
5
|
+
export declare function binary_codec(): Codec<Uint8Array>;
|
|
6
|
+
//# sourceMappingURL=codec.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"codec.d.ts","sourceRoot":"","sources":["../codec.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC;AACrC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAErC,wBAAgB,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAM5D;AAED,wBAAgB,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC,CAM1C;AAED,wBAAgB,YAAY,IAAI,KAAK,CAAC,UAAU,CAAC,CAMhD"}
|