@inixiative/archive 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -47
- package/compose.yaml +24 -3
- package/package.json +11 -5
- package/prisma/migrations/20261001022030_init/migration.sql +119 -0
- package/prisma/migrations/migration_lock.toml +3 -0
- package/prisma/schema.prisma +125 -0
- package/src/agents.ts +5 -15
- package/src/cli.ts +239 -180
- package/src/client.ts +79 -51
- package/src/collector.ts +11 -11
- package/src/config.ts +33 -44
- package/src/kingdom.ts +103 -0
- package/src/remote.ts +170 -0
- package/src/schemas.ts +40 -0
- package/src/server.ts +56 -63
- package/src/store.ts +438 -0
- package/src/local.ts +0 -355
package/src/schemas.ts
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import type { ArchiveSnapshot } from './index';
|
|
3
|
+
import { archiveIntegrationSchema, defaultIntegrations } from './tags';
|
|
4
|
+
|
|
5
|
+
export const archiveSettingsSchema = z.strictObject({
|
|
6
|
+
/** Integrations sessions can reference; Kingdom and Foundry display these, Archive owns them. */
|
|
7
|
+
integrations: z
|
|
8
|
+
.array(archiveIntegrationSchema)
|
|
9
|
+
.max(50)
|
|
10
|
+
.refine(
|
|
11
|
+
(list) => new Set(list.map((i) => i.key)).size === list.length,
|
|
12
|
+
'Duplicate integration keys',
|
|
13
|
+
),
|
|
14
|
+
/** Archives whose latest capture is older than this are deleted here. Deletions never sync. */
|
|
15
|
+
retentionDays: z.number().int().positive().max(36_500).nullable(),
|
|
16
|
+
});
|
|
17
|
+
export type ArchiveSettings = z.infer<typeof archiveSettingsSchema>;
|
|
18
|
+
export const defaultSettings: ArchiveSettings = {
|
|
19
|
+
integrations: defaultIntegrations,
|
|
20
|
+
retentionDays: null,
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export const tagSchema = z.string().min(1).max(120);
|
|
24
|
+
/** A conceptual tag this archive offers: archive-wide (no actor) or for one actor. */
|
|
25
|
+
export const tagDefinitionSchema = z.strictObject({
|
|
26
|
+
tag: tagSchema,
|
|
27
|
+
actorId: z.string().min(1).max(256).optional(),
|
|
28
|
+
description: z.string().max(500).optional(),
|
|
29
|
+
});
|
|
30
|
+
export type TagDefinition = z.infer<typeof tagDefinitionSchema>;
|
|
31
|
+
|
|
32
|
+
export interface ArchiveFilter {
|
|
33
|
+
projectId?: string;
|
|
34
|
+
source?: ArchiveSnapshot['source'];
|
|
35
|
+
tag?: string;
|
|
36
|
+
actorId?: string;
|
|
37
|
+
reference?: { integration: string; ref: string };
|
|
38
|
+
model?: string;
|
|
39
|
+
effort?: string;
|
|
40
|
+
}
|
package/src/server.ts
CHANGED
|
@@ -1,13 +1,7 @@
|
|
|
1
1
|
import { timingSafeEqual } from 'node:crypto';
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
archiveReferenceSchema,
|
|
6
|
-
archiveSnapshotSchema,
|
|
7
|
-
selectChunks,
|
|
8
|
-
snapshotDigest,
|
|
9
|
-
} from './index';
|
|
10
|
-
import { archiveSettingsSchema, LocalArchiveStore, tagDefinitionSchema } from './local';
|
|
3
|
+
import { archiveReferenceSchema, archiveSnapshotSchema, selectChunks } from './index';
|
|
4
|
+
import { ArchiveConflict, ArchiveStore, archiveSettingsSchema, tagDefinitionSchema } from './store';
|
|
11
5
|
|
|
12
6
|
const ingestSchema = z.strictObject({
|
|
13
7
|
snapshot: archiveSnapshotSchema,
|
|
@@ -43,19 +37,8 @@ const tagSchema = z.strictObject({
|
|
|
43
37
|
const tagListSchema = z.strictObject({ actorId: z.string().min(1).max(256).optional() });
|
|
44
38
|
const tagRemoveSchema = tagDefinitionSchema.pick({ tag: true, actorId: true });
|
|
45
39
|
|
|
46
|
-
/** Cursor paging over newest-first archives. */
|
|
47
|
-
function page<T extends { id: string }>(items: T[], limit: number, beforeId?: string) {
|
|
48
|
-
const cursor = beforeId ? items.findIndex((item) => item.id === beforeId) : -1;
|
|
49
|
-
if (beforeId && cursor < 0) return undefined;
|
|
50
|
-
const slice = items.slice(cursor + 1, cursor + 1 + limit);
|
|
51
|
-
return {
|
|
52
|
-
items: slice,
|
|
53
|
-
nextCursor: items.length > cursor + 1 + slice.length ? (slice.at(-1)?.id ?? null) : null,
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
|
|
57
40
|
/** One deployment, one ownership boundary. No shared cross-tenant database or admin API. */
|
|
58
|
-
export function createArchiveHandler(store:
|
|
41
|
+
export function createArchiveHandler(store: ArchiveStore, token: string) {
|
|
59
42
|
if (token.length < 32)
|
|
60
43
|
throw new Error('ARCHIVE_SERVER_TOKEN must contain at least 32 characters');
|
|
61
44
|
const expected = Buffer.from(`Bearer ${token}`);
|
|
@@ -86,48 +69,53 @@ export function createArchiveHandler(store: LocalArchiveStore, token: string) {
|
|
|
86
69
|
chunks.push(item.value);
|
|
87
70
|
}
|
|
88
71
|
const body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
|
72
|
+
if (path === '/api/v1/archive/info') {
|
|
73
|
+
z.strictObject({}).parse(body);
|
|
74
|
+
return json({ data: { sourceId: await store.sourceId(), protocol: 2 } });
|
|
75
|
+
}
|
|
89
76
|
if (path === '/api/v1/archive/ingest') {
|
|
90
77
|
const { snapshot, previousDigest } = ingestSchema.parse(body);
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
if (old && old.digest !== next && old.snapshot.capturedAt > snapshot.capturedAt)
|
|
97
|
-
return json({ error: 'Stale capture' }, 409);
|
|
98
|
-
return json({ data: store.capture(snapshot) });
|
|
78
|
+
return json({ data: await store.capture(snapshot, { previousDigest }) });
|
|
79
|
+
}
|
|
80
|
+
if (path === '/api/v1/archive/head') {
|
|
81
|
+
const head = await store.head(readSchema.pick({ archiveId: true }).parse(body).archiveId);
|
|
82
|
+
return head ? json({ data: head }) : json({ error: 'Archive unavailable' }, 404);
|
|
99
83
|
}
|
|
100
84
|
if (path === '/api/v1/archive/read') {
|
|
101
85
|
const input = readSchema.parse(body),
|
|
102
|
-
archive = store.read(input.archiveId, input.revision);
|
|
86
|
+
archive = await store.read(input.archiveId, input.revision);
|
|
103
87
|
return archive ? json({ data: archive }) : json({ error: 'Archive unavailable' }, 404);
|
|
104
88
|
}
|
|
105
89
|
if (path === '/api/v1/archive/list') {
|
|
106
90
|
const { limit, beforeId, ...filter } = listSchema.parse(body);
|
|
107
|
-
const result =
|
|
91
|
+
const result = await store.page(filter, limit, beforeId);
|
|
108
92
|
if (!result) return json({ error: 'Invalid cursor' }, 400);
|
|
109
93
|
return json({ data: { archives: result.items, nextCursor: result.nextCursor } });
|
|
110
94
|
}
|
|
111
95
|
if (path === '/api/v1/archive/search') {
|
|
112
96
|
const { query, budget, limit, beforeId, ...filter } = searchSchema.parse(body);
|
|
113
|
-
|
|
114
|
-
const
|
|
115
|
-
.
|
|
116
|
-
.
|
|
117
|
-
(a) =>
|
|
118
|
-
|
|
119
|
-
selectChunks(store.read(a.id)!.chunks, query, budget).matchingChunks > 0 ||
|
|
120
|
-
[a.title, ...a.tags].some((text) => text.toLowerCase().includes(query.toLowerCase())),
|
|
97
|
+
const lower = query.toLowerCase();
|
|
98
|
+
const result = await store.page(filter, limit, beforeId, async (listings) => {
|
|
99
|
+
if (!query) return new Set(listings.map((a) => a.id));
|
|
100
|
+
const hits = await store.matching(
|
|
101
|
+
listings.map((a) => a.id),
|
|
102
|
+
query,
|
|
121
103
|
);
|
|
122
|
-
|
|
104
|
+
for (const a of listings)
|
|
105
|
+
if ([a.title, ...a.tags].some((text) => text.toLowerCase().includes(lower)))
|
|
106
|
+
hits.add(a.id);
|
|
107
|
+
return hits;
|
|
108
|
+
});
|
|
123
109
|
if (!result) return json({ error: 'Invalid cursor' }, 400);
|
|
124
|
-
|
|
110
|
+
let remaining = budget;
|
|
111
|
+
const archives = [];
|
|
112
|
+
for (const a of result.items) {
|
|
125
113
|
const selected =
|
|
126
114
|
remaining >= 16
|
|
127
|
-
? selectChunks(store.read(a.id)!.chunks, query, remaining)
|
|
115
|
+
? selectChunks((await store.read(a.id))!.chunks, query, remaining)
|
|
128
116
|
: { chunks: [], tokenCount: 0 };
|
|
129
117
|
remaining -= selected.tokenCount;
|
|
130
|
-
|
|
118
|
+
archives.push({
|
|
131
119
|
archiveId: a.id,
|
|
132
120
|
revision: a.revision,
|
|
133
121
|
digest: a.digest,
|
|
@@ -140,37 +128,43 @@ export function createArchiveHandler(store: LocalArchiveStore, token: string) {
|
|
|
140
128
|
references: a.references,
|
|
141
129
|
coverage: a.coverage,
|
|
142
130
|
...selected,
|
|
143
|
-
};
|
|
144
|
-
}
|
|
131
|
+
});
|
|
132
|
+
}
|
|
145
133
|
return json({
|
|
146
134
|
data: { archives, tokenCount: budget - remaining, nextCursor: result.nextCursor },
|
|
147
135
|
});
|
|
148
136
|
}
|
|
149
137
|
if (path === '/api/v1/archive/tag') {
|
|
150
138
|
const input = tagSchema.parse(body);
|
|
151
|
-
if (!store.read(input.archiveId))
|
|
152
|
-
|
|
139
|
+
if (!(await store.read(input.archiveId)))
|
|
140
|
+
return json({ error: 'Archive unavailable' }, 404);
|
|
141
|
+
return json({ data: await store.tag(input.archiveId, input) });
|
|
153
142
|
}
|
|
154
143
|
if (path === '/api/v1/archive/delete') {
|
|
155
144
|
const input = readSchema.pick({ archiveId: true }).parse(body);
|
|
156
|
-
return store.delete(input.archiveId)
|
|
145
|
+
return (await store.delete(input.archiveId))
|
|
157
146
|
? json({ data: { archiveId: input.archiveId, deleted: true } })
|
|
158
147
|
: json({ error: 'Archive unavailable' }, 404);
|
|
159
148
|
}
|
|
160
149
|
if (path === '/api/v1/archive/settings/read') {
|
|
161
150
|
z.strictObject({}).parse(body);
|
|
162
|
-
return json({ data: store.settings() });
|
|
151
|
+
return json({ data: await store.settings() });
|
|
163
152
|
}
|
|
164
153
|
if (path === '/api/v1/archive/settings/update')
|
|
165
|
-
return json({
|
|
154
|
+
return json({
|
|
155
|
+
data: await store.updateSettings(archiveSettingsSchema.partial().parse(body)),
|
|
156
|
+
});
|
|
166
157
|
if (path === '/api/v1/archive/tags/list')
|
|
167
|
-
return json({
|
|
158
|
+
return json({
|
|
159
|
+
data: { tags: await store.tagDefinitions(tagListSchema.parse(body).actorId) },
|
|
160
|
+
});
|
|
168
161
|
if (path === '/api/v1/archive/tags/define')
|
|
169
|
-
return json({ data: store.defineTag(tagDefinitionSchema.parse(body)) });
|
|
162
|
+
return json({ data: await store.defineTag(tagDefinitionSchema.parse(body)) });
|
|
170
163
|
if (path === '/api/v1/archive/tags/remove')
|
|
171
|
-
return json({ data: { removed: store.undefineTag(tagRemoveSchema.parse(body)) } });
|
|
164
|
+
return json({ data: { removed: await store.undefineTag(tagRemoveSchema.parse(body)) } });
|
|
172
165
|
return json({ error: 'Not found' }, 404);
|
|
173
166
|
} catch (error) {
|
|
167
|
+
if (error instanceof ArchiveConflict) return json({ error: error.message }, 409);
|
|
174
168
|
if (error instanceof z.ZodError || error instanceof SyntaxError)
|
|
175
169
|
return json({ error: 'Invalid archive request' }, 400);
|
|
176
170
|
return json({ error: 'Archive operation failed' }, 500);
|
|
@@ -178,20 +172,20 @@ export function createArchiveHandler(store: LocalArchiveStore, token: string) {
|
|
|
178
172
|
};
|
|
179
173
|
}
|
|
180
174
|
|
|
181
|
-
export function startArchiveServer(options: {
|
|
182
|
-
|
|
175
|
+
export async function startArchiveServer(options: {
|
|
176
|
+
databaseUrl: string;
|
|
183
177
|
token: string;
|
|
184
178
|
port?: number;
|
|
185
179
|
hostname?: string;
|
|
186
180
|
}) {
|
|
187
|
-
const store = new
|
|
188
|
-
// Retention runs at start and hourly; it only ever deletes from this archive.
|
|
189
|
-
store.prune();
|
|
190
|
-
const retention = setInterval(() => store.prune(), 3_600_000);
|
|
191
|
-
retention.unref();
|
|
181
|
+
const store = new ArchiveStore(options.databaseUrl);
|
|
192
182
|
try {
|
|
183
|
+
// Retention runs at start and hourly; it only ever deletes from this archive.
|
|
184
|
+
await store.prune();
|
|
185
|
+
const retention = setInterval(() => void store.prune().catch(() => {}), 3_600_000);
|
|
186
|
+
retention.unref();
|
|
193
187
|
const server = Bun.serve({
|
|
194
|
-
port: options.port ??
|
|
188
|
+
port: options.port ?? 4700,
|
|
195
189
|
hostname: options.hostname ?? '127.0.0.1',
|
|
196
190
|
maxRequestBodySize: 65_000_000,
|
|
197
191
|
fetch: createArchiveHandler(store, options.token),
|
|
@@ -202,12 +196,11 @@ export function startArchiveServer(options: {
|
|
|
202
196
|
async close() {
|
|
203
197
|
clearInterval(retention);
|
|
204
198
|
await server.stop(true);
|
|
205
|
-
store.close();
|
|
199
|
+
await store.close();
|
|
206
200
|
},
|
|
207
201
|
};
|
|
208
202
|
} catch (error) {
|
|
209
|
-
|
|
210
|
-
store.close();
|
|
203
|
+
await store.close();
|
|
211
204
|
throw error;
|
|
212
205
|
}
|
|
213
206
|
}
|
package/src/store.ts
ADDED
|
@@ -0,0 +1,438 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { type Prisma, PrismaClient } from '@prisma/client';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
import {
|
|
5
|
+
type ArchiveChunk,
|
|
6
|
+
type ArchiveSnapshot,
|
|
7
|
+
archiveKey,
|
|
8
|
+
archiveSnapshotSchema,
|
|
9
|
+
chunkArchive,
|
|
10
|
+
selectChunks,
|
|
11
|
+
snapshotDigest,
|
|
12
|
+
} from './index';
|
|
13
|
+
import {
|
|
14
|
+
type ArchiveFilter,
|
|
15
|
+
type ArchiveSettings,
|
|
16
|
+
archiveSettingsSchema,
|
|
17
|
+
defaultSettings,
|
|
18
|
+
type TagDefinition,
|
|
19
|
+
tagDefinitionSchema,
|
|
20
|
+
tagSchema,
|
|
21
|
+
} from './schemas';
|
|
22
|
+
import { archiveReferences, suggestTags } from './tags';
|
|
23
|
+
|
|
24
|
+
export {
|
|
25
|
+
type ArchiveFilter,
|
|
26
|
+
type ArchiveSettings,
|
|
27
|
+
archiveSettingsSchema,
|
|
28
|
+
type TagDefinition,
|
|
29
|
+
tagDefinitionSchema,
|
|
30
|
+
} from './schemas';
|
|
31
|
+
|
|
32
|
+
const summarize = (snapshot: ArchiveSnapshot, settings: ArchiveSettings) => {
|
|
33
|
+
const { entries, tags, ...metadata } = snapshot;
|
|
34
|
+
const models = new Map<string, { model: string; effort?: string; entries: number }>();
|
|
35
|
+
for (const entry of entries) {
|
|
36
|
+
if (!entry.model) continue;
|
|
37
|
+
const key = JSON.stringify([entry.model, entry.effort]);
|
|
38
|
+
const current = models.get(key) ?? {
|
|
39
|
+
model: entry.model,
|
|
40
|
+
...(entry.effort ? { effort: entry.effort } : {}),
|
|
41
|
+
entries: 0,
|
|
42
|
+
};
|
|
43
|
+
current.entries++;
|
|
44
|
+
models.set(key, current);
|
|
45
|
+
}
|
|
46
|
+
return {
|
|
47
|
+
...metadata,
|
|
48
|
+
capturedTags: tags,
|
|
49
|
+
entries: entries.length,
|
|
50
|
+
models: [...models.values()],
|
|
51
|
+
suggestedTags: suggestTags(snapshot),
|
|
52
|
+
references: archiveReferences(snapshot, settings.integrations),
|
|
53
|
+
};
|
|
54
|
+
};
|
|
55
|
+
type Summary = ReturnType<typeof summarize>;
|
|
56
|
+
|
|
57
|
+
const toChunk = (row: {
|
|
58
|
+
id: string;
|
|
59
|
+
entryId: string;
|
|
60
|
+
sourceRef: string;
|
|
61
|
+
kind: string;
|
|
62
|
+
start: number;
|
|
63
|
+
end: number;
|
|
64
|
+
text: string;
|
|
65
|
+
tokenCount: number;
|
|
66
|
+
}): ArchiveChunk => ({
|
|
67
|
+
id: row.id,
|
|
68
|
+
entryId: row.entryId,
|
|
69
|
+
sourceRef: row.sourceRef,
|
|
70
|
+
kind: row.kind as ArchiveChunk['kind'],
|
|
71
|
+
start: row.start,
|
|
72
|
+
end: row.end,
|
|
73
|
+
text: row.text,
|
|
74
|
+
tokenCount: row.tokenCount,
|
|
75
|
+
encoding: 'cl100k_base',
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
const BATCH = 500;
|
|
79
|
+
|
|
80
|
+
type ArchiveRow = {
|
|
81
|
+
id: string;
|
|
82
|
+
revision: number;
|
|
83
|
+
digest: string;
|
|
84
|
+
summary: Prisma.JsonValue;
|
|
85
|
+
tagEdits: { tag: string; added: boolean }[];
|
|
86
|
+
};
|
|
87
|
+
const listing = (row: ArchiveRow, filter: ArchiveFilter) => {
|
|
88
|
+
const { capturedTags, ...summary } = row.summary as Summary;
|
|
89
|
+
const removed = new Set(row.tagEdits.filter((e) => !e.added).map((e) => e.tag));
|
|
90
|
+
const tags = [
|
|
91
|
+
...new Set([...capturedTags, ...row.tagEdits.filter((e) => e.added).map((e) => e.tag)]),
|
|
92
|
+
].filter((tag) => !removed.has(tag));
|
|
93
|
+
const { reference, model, effort } = filter;
|
|
94
|
+
if (
|
|
95
|
+
(filter.tag && !tags.includes(filter.tag)) ||
|
|
96
|
+
(model && !summary.models.some((m) => m.model === model)) ||
|
|
97
|
+
(effort && !summary.models.some((m) => m.effort === effort && (!model || m.model === model))) ||
|
|
98
|
+
(reference &&
|
|
99
|
+
!summary.references.some(
|
|
100
|
+
(r) => r.integration === reference.integration && r.ref === reference.ref,
|
|
101
|
+
))
|
|
102
|
+
)
|
|
103
|
+
return [];
|
|
104
|
+
return [{ id: row.id, revision: row.revision, digest: row.digest, ...summary, tags }];
|
|
105
|
+
};
|
|
106
|
+
export type ArchiveListing = ReturnType<typeof listing>[number];
|
|
107
|
+
|
|
108
|
+
/** One archive on Postgres. One deployment is one ownership boundary. */
|
|
109
|
+
export class ArchiveStore {
|
|
110
|
+
readonly db: PrismaClient;
|
|
111
|
+
private cachedSourceId?: string;
|
|
112
|
+
constructor(databaseUrl: string) {
|
|
113
|
+
this.db = new PrismaClient({ datasourceUrl: databaseUrl });
|
|
114
|
+
}
|
|
115
|
+
/** This archive's identity, stamped into the snapshots its collectors capture. */
|
|
116
|
+
async sourceId() {
|
|
117
|
+
if (this.cachedSourceId) return this.cachedSourceId;
|
|
118
|
+
const row = await this.db.setting.upsert({
|
|
119
|
+
where: { key: 'sourceId' },
|
|
120
|
+
create: { key: 'sourceId', value: randomUUID() },
|
|
121
|
+
update: {},
|
|
122
|
+
});
|
|
123
|
+
this.cachedSourceId = z.uuid().parse(row.value);
|
|
124
|
+
return this.cachedSourceId;
|
|
125
|
+
}
|
|
126
|
+
async capture(input: unknown, expected?: { previousDigest: string | null }) {
|
|
127
|
+
const snapshot = archiveSnapshotSchema.parse(input);
|
|
128
|
+
const id = archiveKey(snapshot);
|
|
129
|
+
const digest = snapshotDigest(snapshot);
|
|
130
|
+
const settings = await this.settings();
|
|
131
|
+
return this.db.$transaction(
|
|
132
|
+
async (tx) => {
|
|
133
|
+
// Serializes concurrent captures of one session.
|
|
134
|
+
await tx.$executeRaw`SELECT 1 FROM pg_advisory_xact_lock(hashtext(${id}))`;
|
|
135
|
+
const old = await tx.archive.findUnique({ where: { id } });
|
|
136
|
+
if (old?.digest === digest) return { id, digest, revision: old.revision, changed: false };
|
|
137
|
+
if (expected && (old?.digest ?? null) !== expected.previousDigest)
|
|
138
|
+
throw new ArchiveConflict('Revision conflict');
|
|
139
|
+
if (old && old.capturedAt > snapshot.capturedAt) throw new ArchiveConflict('Stale capture');
|
|
140
|
+
const revision = (old?.revision ?? 0) + 1;
|
|
141
|
+
const fields = {
|
|
142
|
+
digest,
|
|
143
|
+
revision,
|
|
144
|
+
source: snapshot.source,
|
|
145
|
+
projectId: snapshot.projectId ?? null,
|
|
146
|
+
actorId: snapshot.actor?.id ?? null,
|
|
147
|
+
capturedAt: snapshot.capturedAt,
|
|
148
|
+
summary: summarize(snapshot, settings) as Prisma.InputJsonValue,
|
|
149
|
+
};
|
|
150
|
+
await tx.archive.upsert({ where: { id }, create: { id, ...fields }, update: fields });
|
|
151
|
+
await tx.revision.create({
|
|
152
|
+
data: { archiveId: id, revision, digest, snapshot: snapshot as Prisma.InputJsonValue },
|
|
153
|
+
});
|
|
154
|
+
const chunks = chunkArchive(snapshot);
|
|
155
|
+
for (let offset = 0; offset < chunks.length; offset += 1000)
|
|
156
|
+
await tx.chunk.createMany({
|
|
157
|
+
data: chunks.slice(offset, offset + 1000).map(({ encoding: _, ...chunk }, i) => ({
|
|
158
|
+
archiveId: id,
|
|
159
|
+
revision,
|
|
160
|
+
position: offset + i,
|
|
161
|
+
...chunk,
|
|
162
|
+
})),
|
|
163
|
+
});
|
|
164
|
+
return { id, digest, revision, changed: true };
|
|
165
|
+
// A 64 MB session writes tens of thousands of chunks.
|
|
166
|
+
},
|
|
167
|
+
{ timeout: 120_000, maxWait: 30_000 },
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
/** Current digest and revision without the content. */
|
|
171
|
+
async head(id: string) {
|
|
172
|
+
const row = await this.db.archive.findUnique({
|
|
173
|
+
where: { id },
|
|
174
|
+
select: { id: true, digest: true, revision: true, projectId: true },
|
|
175
|
+
});
|
|
176
|
+
return row ?? undefined;
|
|
177
|
+
}
|
|
178
|
+
async read(id: string, revision?: number) {
|
|
179
|
+
const archive = await this.db.archive.findUnique({ where: { id } });
|
|
180
|
+
if (!archive) return undefined;
|
|
181
|
+
const row = await this.db.revision.findUnique({
|
|
182
|
+
where: { archiveId_revision: { archiveId: id, revision: revision ?? archive.revision } },
|
|
183
|
+
include: { chunks: { orderBy: { position: 'asc' } } },
|
|
184
|
+
});
|
|
185
|
+
if (!row) return undefined;
|
|
186
|
+
return {
|
|
187
|
+
id,
|
|
188
|
+
revision: row.revision,
|
|
189
|
+
digest: row.digest,
|
|
190
|
+
snapshot: archiveSnapshotSchema.parse(row.snapshot),
|
|
191
|
+
chunks: row.chunks.map(toChunk),
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* One page of archive metadata, newest first, read from the database in batches. Tags are the
|
|
196
|
+
* captured tags with this archive's edits applied. `keep` narrows each batch further (search).
|
|
197
|
+
*/
|
|
198
|
+
async page(
|
|
199
|
+
filter: ArchiveFilter,
|
|
200
|
+
limit: number,
|
|
201
|
+
beforeId?: string,
|
|
202
|
+
keep?: (listings: ArchiveListing[]) => Promise<Set<string>>,
|
|
203
|
+
) {
|
|
204
|
+
let cursor: bigint | undefined;
|
|
205
|
+
if (beforeId) {
|
|
206
|
+
const before = await this.db.archive.findUnique({
|
|
207
|
+
where: { id: beforeId },
|
|
208
|
+
select: { seq: true },
|
|
209
|
+
});
|
|
210
|
+
if (!before) return undefined;
|
|
211
|
+
cursor = before.seq;
|
|
212
|
+
}
|
|
213
|
+
const items: ArchiveListing[] = [];
|
|
214
|
+
while (items.length <= limit) {
|
|
215
|
+
const rows = await this.db.archive.findMany({
|
|
216
|
+
where: {
|
|
217
|
+
...(filter.projectId ? { projectId: filter.projectId } : {}),
|
|
218
|
+
...(filter.source ? { source: filter.source } : {}),
|
|
219
|
+
...(filter.actorId ? { actorId: filter.actorId } : {}),
|
|
220
|
+
...(cursor === undefined ? {} : { seq: { lt: cursor } }),
|
|
221
|
+
},
|
|
222
|
+
include: { tagEdits: true },
|
|
223
|
+
orderBy: { seq: 'desc' },
|
|
224
|
+
take: BATCH,
|
|
225
|
+
});
|
|
226
|
+
if (!rows.length) break;
|
|
227
|
+
cursor = rows.at(-1)!.seq;
|
|
228
|
+
let listings = rows.flatMap((row) => listing(row, filter));
|
|
229
|
+
if (keep && listings.length) {
|
|
230
|
+
const kept = await keep(listings);
|
|
231
|
+
listings = listings.filter((item) => kept.has(item.id));
|
|
232
|
+
}
|
|
233
|
+
items.push(...listings);
|
|
234
|
+
if (rows.length < BATCH) break;
|
|
235
|
+
}
|
|
236
|
+
const slice = items.slice(0, limit);
|
|
237
|
+
return { items: slice, nextCursor: items.length > limit ? slice.at(-1)!.id : null };
|
|
238
|
+
}
|
|
239
|
+
/** Every archive matching the filter, newest first. */
|
|
240
|
+
async list(filter: ArchiveFilter = {}) {
|
|
241
|
+
const all: ArchiveListing[] = [];
|
|
242
|
+
let beforeId: string | undefined;
|
|
243
|
+
do {
|
|
244
|
+
const result = (await this.page(filter, BATCH, beforeId))!;
|
|
245
|
+
all.push(...result.items);
|
|
246
|
+
beforeId = result.nextCursor ?? undefined;
|
|
247
|
+
} while (beforeId);
|
|
248
|
+
return all;
|
|
249
|
+
}
|
|
250
|
+
/** Ids among `ids` whose current revision has a chunk containing any term. */
|
|
251
|
+
async matching(ids: string[], query: string) {
|
|
252
|
+
const terms = [...new Set(query.toLowerCase().match(/[\p{L}\p{N}_-]+/gu) ?? [])].slice(0, 64);
|
|
253
|
+
if (!terms.length || !ids.length) return new Set(ids);
|
|
254
|
+
const current = await this.db.archive.findMany({
|
|
255
|
+
where: { id: { in: ids } },
|
|
256
|
+
select: { id: true, revision: true },
|
|
257
|
+
});
|
|
258
|
+
const rows = await this.db.chunk.findMany({
|
|
259
|
+
where: {
|
|
260
|
+
OR: current.map((a) => ({ archiveId: a.id, revision: a.revision })),
|
|
261
|
+
AND: {
|
|
262
|
+
OR: terms.map((term) => ({ text: { contains: term, mode: 'insensitive' as const } })),
|
|
263
|
+
},
|
|
264
|
+
},
|
|
265
|
+
distinct: ['archiveId'],
|
|
266
|
+
select: { archiveId: true },
|
|
267
|
+
});
|
|
268
|
+
return new Set(rows.map((row) => row.archiveId));
|
|
269
|
+
}
|
|
270
|
+
async search(ids: string[], query: string, budget = 2048) {
|
|
271
|
+
return Promise.all(
|
|
272
|
+
ids.map(async (id) => {
|
|
273
|
+
const archive = await this.read(id);
|
|
274
|
+
if (!archive) throw new Error('Archive unavailable');
|
|
275
|
+
return {
|
|
276
|
+
id,
|
|
277
|
+
revision: archive.revision,
|
|
278
|
+
digest: archive.digest,
|
|
279
|
+
...selectChunks(archive.chunks, query, budget),
|
|
280
|
+
};
|
|
281
|
+
}),
|
|
282
|
+
);
|
|
283
|
+
}
|
|
284
|
+
/** Tag or untag without a new revision. Edits belong to this archive and do not sync. */
|
|
285
|
+
async tag(id: string, change: { add?: string[]; remove?: string[] }) {
|
|
286
|
+
const add = z
|
|
287
|
+
.array(tagSchema)
|
|
288
|
+
.max(100)
|
|
289
|
+
.parse(change.add ?? []);
|
|
290
|
+
const remove = z
|
|
291
|
+
.array(tagSchema)
|
|
292
|
+
.max(100)
|
|
293
|
+
.parse(change.remove ?? []);
|
|
294
|
+
return this.db.$transaction(async (tx) => {
|
|
295
|
+
const archive = await tx.archive.findUnique({ where: { id } });
|
|
296
|
+
if (!archive) throw new Error('Archive unavailable');
|
|
297
|
+
const captured = new Set((archive.summary as Summary).capturedTags);
|
|
298
|
+
const set = (tag: string, added: boolean) =>
|
|
299
|
+
tx.tagEdit.upsert({
|
|
300
|
+
where: { archiveId_tag: { archiveId: id, tag } },
|
|
301
|
+
create: { archiveId: id, tag, added },
|
|
302
|
+
update: { added },
|
|
303
|
+
});
|
|
304
|
+
const clear = (tag: string) => tx.tagEdit.deleteMany({ where: { archiveId: id, tag } });
|
|
305
|
+
for (const tag of add) await (captured.has(tag) ? clear(tag) : set(tag, true));
|
|
306
|
+
for (const tag of remove) await (captured.has(tag) ? set(tag, false) : clear(tag));
|
|
307
|
+
const edits = await tx.tagEdit.findMany({ where: { archiveId: id } });
|
|
308
|
+
const removed = new Set(edits.filter((e) => !e.added).map((e) => e.tag));
|
|
309
|
+
const tags = [
|
|
310
|
+
...new Set([...captured, ...edits.filter((e) => e.added).map((e) => e.tag)]),
|
|
311
|
+
].filter((tag) => !removed.has(tag));
|
|
312
|
+
z.array(tagSchema).max(100, 'An archive holds at most 100 tags').parse(tags);
|
|
313
|
+
return { id, tags };
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
async delete(id: string) {
|
|
317
|
+
return (await this.db.archive.deleteMany({ where: { id } })).count > 0;
|
|
318
|
+
}
|
|
319
|
+
/** Applies the retention setting; returns the deleted archive ids. */
|
|
320
|
+
async prune(now = Date.now()) {
|
|
321
|
+
const { retentionDays } = await this.settings();
|
|
322
|
+
if (retentionDays === null) return [];
|
|
323
|
+
const expired = await this.db.archive.findMany({
|
|
324
|
+
where: { capturedAt: { lt: now - retentionDays * 86_400_000 } },
|
|
325
|
+
select: { id: true },
|
|
326
|
+
});
|
|
327
|
+
await this.db.archive.deleteMany({ where: { id: { in: expired.map((a) => a.id) } } });
|
|
328
|
+
return expired.map((a) => a.id);
|
|
329
|
+
}
|
|
330
|
+
async settings(): Promise<ArchiveSettings> {
|
|
331
|
+
const row = await this.db.setting.findUnique({ where: { key: 'archive' } });
|
|
332
|
+
return row ? archiveSettingsSchema.parse(row.value) : defaultSettings;
|
|
333
|
+
}
|
|
334
|
+
/** Saves settings and recomputes listings together, since references follow the integrations. */
|
|
335
|
+
async updateSettings(change: Partial<ArchiveSettings>) {
|
|
336
|
+
const next = archiveSettingsSchema.parse({ ...(await this.settings()), ...change });
|
|
337
|
+
await this.db.$transaction(
|
|
338
|
+
async (tx) => {
|
|
339
|
+
await tx.setting.upsert({
|
|
340
|
+
where: { key: 'archive' },
|
|
341
|
+
create: { key: 'archive', value: next },
|
|
342
|
+
update: { value: next },
|
|
343
|
+
});
|
|
344
|
+
const archives = await tx.archive.findMany({ select: { id: true, revision: true } });
|
|
345
|
+
for (const archive of archives) {
|
|
346
|
+
const row = await tx.revision.findUnique({
|
|
347
|
+
where: { archiveId_revision: { archiveId: archive.id, revision: archive.revision } },
|
|
348
|
+
select: { snapshot: true },
|
|
349
|
+
});
|
|
350
|
+
if (!row) continue;
|
|
351
|
+
await tx.archive.update({
|
|
352
|
+
where: { id: archive.id },
|
|
353
|
+
data: {
|
|
354
|
+
summary: summarize(
|
|
355
|
+
archiveSnapshotSchema.parse(row.snapshot),
|
|
356
|
+
next,
|
|
357
|
+
) as Prisma.InputJsonValue,
|
|
358
|
+
},
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
},
|
|
362
|
+
{ timeout: 600_000, maxWait: 30_000 },
|
|
363
|
+
);
|
|
364
|
+
return next;
|
|
365
|
+
}
|
|
366
|
+
/** Offered tags with how many archives carry each; an actor sees archive-wide and their own. */
|
|
367
|
+
async tagDefinitions(actorId?: string) {
|
|
368
|
+
const rows = await this.db.tagDefinition.findMany({
|
|
369
|
+
where: { actorId: { in: ['', actorId ?? ''] } },
|
|
370
|
+
orderBy: { tag: 'asc' },
|
|
371
|
+
});
|
|
372
|
+
const archives = await this.list(actorId ? { actorId } : {});
|
|
373
|
+
return rows.map((row) => ({
|
|
374
|
+
tag: row.tag,
|
|
375
|
+
...(row.actorId ? { actorId: row.actorId } : {}),
|
|
376
|
+
...(row.description ? { description: row.description } : {}),
|
|
377
|
+
archives: archives.filter((archive) => archive.tags.includes(row.tag)).length,
|
|
378
|
+
}));
|
|
379
|
+
}
|
|
380
|
+
async defineTag(input: TagDefinition) {
|
|
381
|
+
const definition = tagDefinitionSchema.parse(input);
|
|
382
|
+
const actorId = definition.actorId ?? '';
|
|
383
|
+
const description = definition.description ?? null;
|
|
384
|
+
await this.db.tagDefinition.upsert({
|
|
385
|
+
where: { actorId_tag: { actorId, tag: definition.tag } },
|
|
386
|
+
create: { actorId, tag: definition.tag, description },
|
|
387
|
+
update: { description },
|
|
388
|
+
});
|
|
389
|
+
return definition;
|
|
390
|
+
}
|
|
391
|
+
async undefineTag(input: { tag: string; actorId?: string }) {
|
|
392
|
+
return (
|
|
393
|
+
(
|
|
394
|
+
await this.db.tagDefinition.deleteMany({
|
|
395
|
+
where: { actorId: input.actorId ?? '', tag: input.tag },
|
|
396
|
+
})
|
|
397
|
+
).count > 0
|
|
398
|
+
);
|
|
399
|
+
}
|
|
400
|
+
async receipt(id: string, destination: string) {
|
|
401
|
+
return (
|
|
402
|
+
(
|
|
403
|
+
await this.db.receipt.findUnique({
|
|
404
|
+
where: { archiveId_destination: { archiveId: id, destination } },
|
|
405
|
+
})
|
|
406
|
+
)?.digest ?? null
|
|
407
|
+
);
|
|
408
|
+
}
|
|
409
|
+
async pending(id: string, destination: string) {
|
|
410
|
+
const row = await this.db.outbox.findUnique({
|
|
411
|
+
where: { archiveId_destination: { archiveId: id, destination } },
|
|
412
|
+
});
|
|
413
|
+
return row ? { revision: row.revision } : null;
|
|
414
|
+
}
|
|
415
|
+
async enqueue(id: string, destination: string, pending: { revision: number }) {
|
|
416
|
+
await this.db.outbox.upsert({
|
|
417
|
+
where: { archiveId_destination: { archiveId: id, destination } },
|
|
418
|
+
create: { archiveId: id, destination, revision: pending.revision },
|
|
419
|
+
update: {},
|
|
420
|
+
});
|
|
421
|
+
}
|
|
422
|
+
async delivered(id: string, destination: string, digest: string) {
|
|
423
|
+
await this.db.$transaction([
|
|
424
|
+
this.db.receipt.upsert({
|
|
425
|
+
where: { archiveId_destination: { archiveId: id, destination } },
|
|
426
|
+
create: { archiveId: id, destination, digest },
|
|
427
|
+
update: { digest },
|
|
428
|
+
}),
|
|
429
|
+
this.db.outbox.deleteMany({ where: { archiveId: id, destination } }),
|
|
430
|
+
]);
|
|
431
|
+
}
|
|
432
|
+
async close() {
|
|
433
|
+
await this.db.$disconnect();
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/** A capture that lost a race or arrived out of order; the server answers 409. */
|
|
438
|
+
export class ArchiveConflict extends Error {}
|