@inixiative/archive 0.5.1 → 0.7.0

package/src/index.ts

@@ -11,6 +11,22 @@ export const archiveEntrySchema = z.strictObject({
   text: z.string().max(2_000_000),
   timestamp: z.number().finite().nonnegative().nullable(),
   sourceRef: z.string().min(1).max(512),
+  /** The model that produced the entry, and its effort level as the harness names it. */
+  model: z.string().min(1).max(200).optional(),
+  effort: z.string().min(1).max(40).optional(),
+});
+/** Who did the work: a person, a service such as an API-key run, or a job (id = job name). */
+export const archiveActorSchema = z.strictObject({
+  kind: z.enum(['user', 'service', 'job']),
+  id: identifier,
+  name: z.string().min(1).max(200).optional(),
+  /** The integration the work ran under, when there is one. */
+  integrationId: identifier.optional(),
+});
+/** An item in an integration (a GitHub pull request, a Linear issue) the session is about. */
+export const archiveReferenceSchema = z.strictObject({
+  integration: z.string().regex(/^[a-z0-9-]{1,64}$/),
+  ref: z.string().min(1).max(300),
 });
 export const archiveSnapshotSchema = z
   .strictObject({
@@ -20,6 +36,9 @@ export const archiveSnapshotSchema = z
     sessionId: identifier,
     title: z.string().min(1).max(500),
     projectId: identifier.optional(),
+    actor: archiveActorSchema.optional(),
+    /** Recorded references, such as the session's repository and branch. Linked ones are found in the text. */
+    references: z.array(archiveReferenceSchema).max(100).optional(),
     tags: z.array(z.string().min(1).max(120)).max(100),
     goalIds: z.array(identifier).max(100).default([]),
     runIds: z.array(identifier).max(100).default([]),
@@ -39,6 +58,8 @@ export const archiveSnapshotSchema = z
   });
 export type ArchiveSnapshot = z.infer<typeof archiveSnapshotSchema>;
 export type ArchiveEntry = z.infer<typeof archiveEntrySchema>;
+export type ArchiveActor = z.infer<typeof archiveActorSchema>;
+export type ArchiveReference = z.infer<typeof archiveReferenceSchema>;
 export interface ArchiveChunk {
   id: string;
   entryId: string;

package/src/remote.ts

@@ -0,0 +1,166 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { defaultHome } from './agents';
+import type { ArchiveChunk, ArchiveSnapshot } from './index';
+import { archiveKey, archiveSnapshotSchema } from './index';
+import type { ArchiveFilter, ArchiveSettings, TagDefinition } from './schemas';
+import type { ArchiveReferenceCount, TagSuggestion } from './tags';
+
+export type ArchiveListing = Omit<ArchiveSnapshot, 'entries'> & {
+  id: string;
+  revision: number;
+  digest: string;
+  entries: number;
+  models: { model: string; effort?: string; entries: number }[];
+  suggestedTags: TagSuggestion[];
+  references: ArchiveReferenceCount[];
+};
+
+export class ArchiveRequestError extends Error {
+  constructor(
+    readonly status: number,
+    message: string,
+  ) {
+    super(message);
+  }
+}
+
+/**
+ * An Archive server over HTTP. Collectors, the CLI and Foundry write through this; only the
+ * server holds the database.
+ */
+export class ArchiveClient {
+  private cachedSourceId?: string;
+  constructor(private readonly options: { url: string; token: string; fetch?: typeof fetch }) {}
+  async request<T>(action: string, body: unknown = {}): Promise<T> {
+    const response = await (this.options.fetch ?? fetch)(
+      new URL(`api/v1/archive/${action}`, this.options.url),
+      {
+        method: 'POST',
+        redirect: 'error',
+        signal: AbortSignal.timeout(30_000),
+        headers: {
+          authorization: `Bearer ${this.options.token}`,
+          'content-type': 'application/json',
+        },
+        body: JSON.stringify(body),
+      },
+    );
+    const payload = (await response.json().catch(() => ({}))) as { data?: T; error?: string };
+    if (!response.ok)
+      throw new ArchiveRequestError(response.status, payload.error ?? `Archive ${action} failed`);
+    return payload.data as T;
+  }
+  async sourceId() {
+    this.cachedSourceId ??= (await this.request<{ sourceId: string }>('info')).sourceId;
+    return this.cachedSourceId;
+  }
+  head(id: string) {
+    return this.request<{ id: string; digest: string; revision: number; projectId: string | null }>(
+      'head',
+      { archiveId: id },
+    ).catch((error) => {
+      if (error instanceof ArchiveRequestError && error.status === 404) return undefined;
+      throw error;
+    });
+  }
+  /** Captures a snapshot as the next revision, retrying when another writer got there first. */
+  async capture(input: unknown) {
+    const snapshot = archiveSnapshotSchema.parse(input);
+    const id = archiveKey(snapshot);
+    for (let attempt = 0; ; attempt++) {
+      const head = await this.head(id);
+      try {
+        return await this.request<{
+          id: string;
+          digest: string;
+          revision: number;
+          changed: boolean;
+        }>('ingest', { snapshot, previousDigest: head?.digest ?? null });
+      } catch (error) {
+        const conflict =
+          error instanceof ArchiveRequestError &&
+          error.status === 409 &&
+          error.message === 'Revision conflict';
+        if (!conflict || attempt >= 3) throw error;
+      }
+    }
+  }
+  read(id: string, revision?: number) {
+    return this.request<{
+      id: string;
+      revision: number;
+      digest: string;
+      snapshot: ArchiveSnapshot;
+      chunks: ArchiveChunk[];
+    }>('read', { archiveId: id, ...(revision ? { revision } : {}) }).catch((error) => {
+      if (error instanceof ArchiveRequestError && error.status === 404) return undefined;
+      throw error;
+    });
+  }
+  /** Every listing matching the filter, following cursors. */
+  async list(filter: ArchiveFilter = {}) {
+    const archives: ArchiveListing[] = [];
+    let beforeId: string | undefined;
+    do {
+      const page = await this.request<{ archives: ArchiveListing[]; nextCursor: string | null }>(
+        'list',
+        { ...filter, limit: 100, ...(beforeId ? { beforeId } : {}) },
+      );
+      archives.push(...page.archives);
+      beforeId = page.nextCursor ?? undefined;
+    } while (beforeId);
+    return archives;
+  }
+  search(
+    body: ArchiveFilter & { query?: string; budget?: number; limit?: number; beforeId?: string },
+  ) {
+    return this.request<{
+      archives: (Omit<ArchiveListing, 'id' | 'entries'> & {
+        archiveId: string;
+        chunks: ArchiveChunk[];
+        tokenCount: number;
+      })[];
+      tokenCount: number;
+      nextCursor: string | null;
+    }>('search', body);
+  }
+  tag(id: string, change: { add?: string[]; remove?: string[] }) {
+    return this.request<{ id: string; tags: string[] }>('tag', { archiveId: id, ...change });
+  }
+  delete(id: string) {
+    return this.request<{ deleted: boolean }>('delete', { archiveId: id });
+  }
+  settings() {
+    return this.request<ArchiveSettings>('settings/read');
+  }
+  updateSettings(change: Partial<ArchiveSettings>) {
+    return this.request<ArchiveSettings>('settings/update', change);
+  }
+  tagDefinitions(actorId?: string) {
+    return this.request<{ tags: (TagDefinition & { archives: number })[] }>(
+      'tags/list',
+      actorId ? { actorId } : {},
+    );
+  }
+}
+
+/** The machine's local Archive server (the `archive` block in @inixiative/config ports). */
+export const DEFAULT_URL = 'http://127.0.0.1:4700';
+
+/**
+ * A client for this machine's Archive: `ARCHIVE_URL` or the default address, with `ARCHIVE_TOKEN`
+ * or the home's `server.token`. Undefined until `archive up` (or `init`) has made a token.
+ */
+export function localArchive(options: { home?: string; url?: string; fetch?: typeof fetch } = {}) {
+  const tokenFile = join(options.home ?? defaultHome(), 'server.token');
+  const token =
+    process.env.ARCHIVE_TOKEN ??
+    (existsSync(tokenFile) ? readFileSync(tokenFile, 'utf8').trim() : undefined);
+  if (!token) return undefined;
+  return new ArchiveClient({
+    url: options.url ?? process.env.ARCHIVE_URL ?? DEFAULT_URL,
+    token,
+    fetch: options.fetch,
+  });
+}

package/src/schemas.ts

@@ -0,0 +1,40 @@
+import { z } from 'zod';
+import type { ArchiveSnapshot } from './index';
+import { archiveIntegrationSchema, defaultIntegrations } from './tags';
+
+export const archiveSettingsSchema = z.strictObject({
+  /** Integrations sessions can reference; Kingdom and Foundry display these, Archive owns them. */
+  integrations: z
+    .array(archiveIntegrationSchema)
+    .max(50)
+    .refine(
+      (list) => new Set(list.map((i) => i.key)).size === list.length,
+      'Duplicate integration keys',
+    ),
+  /** Archives whose latest capture is older than this are deleted here. Deletions never sync. */
+  retentionDays: z.number().int().positive().max(36_500).nullable(),
+});
+export type ArchiveSettings = z.infer<typeof archiveSettingsSchema>;
+export const defaultSettings: ArchiveSettings = {
+  integrations: defaultIntegrations,
+  retentionDays: null,
+};
+
+export const tagSchema = z.string().min(1).max(120);
+/** A conceptual tag this archive offers: archive-wide (no actor) or for one actor. */
+export const tagDefinitionSchema = z.strictObject({
+  tag: tagSchema,
+  actorId: z.string().min(1).max(256).optional(),
+  description: z.string().max(500).optional(),
+});
+export type TagDefinition = z.infer<typeof tagDefinitionSchema>;
+
+export interface ArchiveFilter {
+  projectId?: string;
+  source?: ArchiveSnapshot['source'];
+  tag?: string;
+  actorId?: string;
+  reference?: { integration: string; ref: string };
+  model?: string;
+  effort?: string;
+}

package/src/server.ts

@@ -1,7 +1,7 @@
 import { timingSafeEqual } from 'node:crypto';
 import { z } from 'zod';
-import { archiveKey, archiveSnapshotSchema, selectChunks, snapshotDigest } from './index';
-import { LocalArchiveStore } from './local';
+import { archiveReferenceSchema, archiveSnapshotSchema, selectChunks } from './index';
+import { ArchiveConflict, ArchiveStore, archiveSettingsSchema, tagDefinitionSchema } from './store';
 
 const ingestSchema = z.strictObject({
   snapshot: archiveSnapshotSchema,
@@ -10,21 +10,35 @@ const ingestSchema = z.strictObject({
     .regex(/^[a-f0-9]{64}$/)
     .nullable(),
 });
+const archiveId = z.string().regex(/^[a-f0-9]{64}$/);
+const filterFields = {
+  projectId: z.string().max(256).optional(),
+  source: archiveSnapshotSchema.shape.source.optional(),
+  tag: z.string().min(1).max(120).optional(),
+  actorId: z.string().min(1).max(256).optional(),
+  reference: archiveReferenceSchema.optional(),
+  model: z.string().min(1).max(200).optional(),
+  effort: z.string().min(1).max(40).optional(),
+  limit: z.number().int().min(1).max(100).default(20),
+  beforeId: archiveId.optional(),
+};
+const listSchema = z.strictObject(filterFields);
 const searchSchema = z.strictObject({
+  ...filterFields,
   query: z.string().max(1000).default(''),
-  tag: z.string().min(1).max(120).optional(),
   budget: z.number().int().min(16).max(32768).default(2048),
-  projectId: z.string().max(256).optional(),
-  limit: z.number().int().min(1).max(100).default(20),
-  beforeId: z.string().optional(),
 });
-const readSchema = z.strictObject({
-  archiveId: z.string().regex(/^[a-f0-9]{64}$/),
-  revision: z.number().int().positive().optional(),
+const readSchema = z.strictObject({ archiveId, revision: z.number().int().positive().optional() });
+const tagSchema = z.strictObject({
+  archiveId,
+  add: z.array(z.string()).max(100).optional(),
+  remove: z.array(z.string()).max(100).optional(),
 });
+const tagListSchema = z.strictObject({ actorId: z.string().min(1).max(256).optional() });
+const tagRemoveSchema = tagDefinitionSchema.pick({ tag: true, actorId: true });
 
 /** One deployment, one ownership boundary. No shared cross-tenant database or admin API. */
-export function createArchiveHandler(store: LocalArchiveStore, token: string) {
+export function createArchiveHandler(store: ArchiveStore, token: string) {
   if (token.length < 32)
     throw new Error('ARCHIVE_SERVER_TOKEN must contain at least 32 characters');
   const expected = Buffer.from(`Bearer ${token}`);
@@ -55,75 +69,102 @@ export function createArchiveHandler(store: LocalArchiveStore, token: string) {
         chunks.push(item.value);
       }
       const body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+      if (path === '/api/v1/archive/info') {
+        z.strictObject({}).parse(body);
+        return json({ data: { sourceId: await store.sourceId(), protocol: 2 } });
+      }
       if (path === '/api/v1/archive/ingest') {
         const { snapshot, previousDigest } = ingestSchema.parse(body);
-        const id = archiveKey(snapshot),
-          old = store.read(id),
-          next = snapshotDigest(snapshot);
-        if (old?.digest !== next && (old?.digest ?? null) !== previousDigest)
-          return json({ error: 'Revision conflict' }, 409);
-        if (old && old.digest !== next && old.snapshot.capturedAt > snapshot.capturedAt)
-          return json({ error: 'Stale capture' }, 409);
-        return json({ data: store.capture(snapshot) });
+        return json({ data: await store.capture(snapshot, { previousDigest }) });
+      }
+      if (path === '/api/v1/archive/head') {
+        const head = await store.head(readSchema.pick({ archiveId: true }).parse(body).archiveId);
+        return head ? json({ data: head }) : json({ error: 'Archive unavailable' }, 404);
       }
       if (path === '/api/v1/archive/read') {
         const input = readSchema.parse(body),
-          archive = store.read(input.archiveId, input.revision);
+          archive = await store.read(input.archiveId, input.revision);
         return archive ? json({ data: archive }) : json({ error: 'Archive unavailable' }, 404);
       }
+      if (path === '/api/v1/archive/list') {
+        const { limit, beforeId, ...filter } = listSchema.parse(body);
+        const result = await store.page(filter, limit, beforeId);
+        if (!result) return json({ error: 'Invalid cursor' }, 400);
+        return json({ data: { archives: result.items, nextCursor: result.nextCursor } });
+      }
       if (path === '/api/v1/archive/search') {
-        const input = searchSchema.parse(body);
-        let remaining = input.budget;
-        const matching = store
-          .list()
-          .filter(
-            (a) =>
-              (!input.projectId || a.projectId === input.projectId) &&
-              (!input.tag || a.tags.includes(input.tag)),
-          )
-          .map((a) => ({
-            a,
-            selected: selectChunks(store.read(a.id)!.chunks, input.query, input.budget),
-          }))
-          .filter(
-            ({ a, selected }) =>
-              !input.query ||
-              selected.matchingChunks > 0 ||
-              [a.title, ...a.tags].some((text) =>
-                text.toLowerCase().includes(input.query.toLowerCase()),
-              ),
+        const { query, budget, limit, beforeId, ...filter } = searchSchema.parse(body);
+        const lower = query.toLowerCase();
+        const result = await store.page(filter, limit, beforeId, async (listings) => {
+          if (!query) return new Set(listings.map((a) => a.id));
+          const hits = await store.matching(
+            listings.map((a) => a.id),
+            query,
           );
-        const cursor = input.beforeId ? matching.findIndex(({ a }) => a.id === input.beforeId) : -1;
-        if (input.beforeId && cursor < 0) return json({ error: 'Invalid cursor' }, 400);
-        const page = matching.slice(cursor + 1, cursor + 1 + input.limit);
-        const archives = page.map(({ a }) => {
+          for (const a of listings)
+            if ([a.title, ...a.tags].some((text) => text.toLowerCase().includes(lower)))
+              hits.add(a.id);
+          return hits;
+        });
+        if (!result) return json({ error: 'Invalid cursor' }, 400);
+        let remaining = budget;
+        const archives = [];
+        for (const a of result.items) {
           const selected =
             remaining >= 16
-              ? selectChunks(store.read(a.id)!.chunks, input.query, remaining)
+              ? selectChunks((await store.read(a.id))!.chunks, query, remaining)
               : { chunks: [], tokenCount: 0 };
           remaining -= selected.tokenCount;
-          return {
+          archives.push({
             archiveId: a.id,
             revision: a.revision,
             digest: a.digest,
             title: a.title,
             projectId: a.projectId,
+            actor: a.actor,
+            models: a.models,
             tags: a.tags,
             suggestedTags: a.suggestedTags,
+            references: a.references,
             coverage: a.coverage,
             ...selected,
-          };
-        });
+          });
+        }
         return json({
-          data: {
-            archives,
-            tokenCount: input.budget - remaining,
-            nextCursor: matching.length > cursor + 1 + page.length ? page.at(-1)?.a.id : null,
-          },
+          data: { archives, tokenCount: budget - remaining, nextCursor: result.nextCursor },
         });
       }
+      if (path === '/api/v1/archive/tag') {
+        const input = tagSchema.parse(body);
+        if (!(await store.read(input.archiveId)))
+          return json({ error: 'Archive unavailable' }, 404);
+        return json({ data: await store.tag(input.archiveId, input) });
+      }
+      if (path === '/api/v1/archive/delete') {
+        const input = readSchema.pick({ archiveId: true }).parse(body);
+        return (await store.delete(input.archiveId))
+          ? json({ data: { archiveId: input.archiveId, deleted: true } })
+          : json({ error: 'Archive unavailable' }, 404);
+      }
+      if (path === '/api/v1/archive/settings/read') {
+        z.strictObject({}).parse(body);
+        return json({ data: await store.settings() });
+      }
+      if (path === '/api/v1/archive/settings/update')
+        return json({
+          data: await store.updateSettings(archiveSettingsSchema.partial().parse(body)),
+        });
+      if (path === '/api/v1/archive/tags/list')
+        return json({
+          data: { tags: await store.tagDefinitions(tagListSchema.parse(body).actorId) },
+        });
+      if (path === '/api/v1/archive/tags/define')
+        return json({ data: await store.defineTag(tagDefinitionSchema.parse(body)) });
+      if (path === '/api/v1/archive/tags/remove')
+        return json({ data: { removed: await store.undefineTag(tagRemoveSchema.parse(body)) } });
       return json({ error: 'Not found' }, 404);
     } catch (error) {
+      if (error instanceof ArchiveConflict) return json({ error: error.message }, 409);
       if (error instanceof z.ZodError || error instanceof SyntaxError)
         return json({ error: 'Invalid archive request' }, 400);
       return json({ error: 'Archive operation failed' }, 500);
@@ -131,16 +172,20 @@ export function createArchiveHandler(store: LocalArchiveStore, token: string) {
   };
 }
 
-export function startArchiveServer(options: {
-  store: string;
+export async function startArchiveServer(options: {
+  databaseUrl: string;
   token: string;
   port?: number;
   hostname?: string;
 }) {
-  const store = new LocalArchiveStore(options.store);
+  const store = new ArchiveStore(options.databaseUrl);
   try {
+    // Retention runs at start and hourly; it only ever deletes from this archive.
+    await store.prune();
+    const retention = setInterval(() => void store.prune().catch(() => {}), 3_600_000);
+    retention.unref();
     const server = Bun.serve({
-      port: options.port ?? 4411,
+      port: options.port ?? 4700,
       hostname: options.hostname ?? '127.0.0.1',
       maxRequestBodySize: 65_000_000,
       fetch: createArchiveHandler(store, options.token),
@@ -149,12 +194,13 @@ export function startArchiveServer(options: {
       server,
       store,
       async close() {
+        clearInterval(retention);
         await server.stop(true);
-        store.close();
+        await store.close();
       },
     };
   } catch (error) {
-    store.close();
+    await store.close();
     throw error;
   }
 }