@volter/twin-openai 0.1.2 → 2.0.0

package/src/semantics/vector-stores.ts

@@ -0,0 +1,279 @@
+// Vector store semantics: the store, the files attached to it, file batches, and search. List,
+// retrieve and delete of a store and retrieve of a file batch are the derived core's. An attached file
+// and a file batch are created `in_progress`, as OpenAI creates them; the twin embeds nothing, so a
+// read that first looks at them finds them `completed` (./progress.ts). An attached
+// file is stored as 'vector_store_file' under `<store>::<file>` (one file may sit in several stores)
+// and shown with the file's id by the manifest's view; its operations are handlers, because OpenAI
+// names a missing one by both ids and answers its deletion with the file's.
+import type { Semantics, SemanticsContext } from '@volter/world-core';
+import { coreAnswer, progress, progressAll, readOnly } from './progress.ts';
+import { at, epoch, invalid, missing, objectOr, page, type Row } from './shared.ts';
+
+const STORE = 'VectorStoreObject';
+const FILE = 'VectorStoreFileObject';
+const BATCH = 'VectorStoreFileBatchObject';
+
+async function attach(ctx: SemanticsContext, storeId: string, fileId: string, batchId?: string, attributes?: unknown): Promise<Row> {
+  // a file detached earlier is attached anew; its attributes are what search filters read
+  // (https://platform.openai.com/docs/api-reference/vector-stores-files/createFile, `attributes`)
+  const fields = { object: 'vector_store.file', vector_store_id: storeId, file_id: fileId, created_at: epoch(ctx), status: 'in_progress', usage_bytes: 0, ...(attributes && typeof attributes === 'object' ? { attributes } : {}), deleted: false, ...(batchId ? { _batch_id: batchId } : {}) };
+  return ctx.write(FILE, `${storeId}::${fileId}`, fields, 'vector_store_file.create');
+}
+
+/** What an indexed file carries: the bytes it uses in the store, its File's size (the twin chunks and embeds nothing, so
+ *  no more and no less: the spec's VectorStoreFileObject `usage_bytes`, "The total vector store usage in bytes"). */
+const indexed = (ctx: SemanticsContext) => (row: Row, end: string): Row => (end === 'completed' ? { usage_bytes: Number(ctx.row('OpenAIFile', String(row.file_id), { withDeleted: true })?.bytes ?? 0) } : {});
+
+/** A store's size, the bytes its files use (the spec's VectorStoreObject `usage_bytes`: "The total number of bytes used by
+ *  the files in the vector store."; `bytes`, which the reference's examples answer beside it, is the same size), written
+ *  when it changes. */
+async function recount(ctx: SemanticsContext, storeId: string): Promise<void> {
+  const row = ctx.row(STORE, storeId, { withDeleted: true });
+  if (!row || row.deleted) return;
+  const used = attached(ctx, storeId).reduce((n, f) => n + Number(f.usage_bytes ?? 0), 0);
+  if (Number(row.usage_bytes) !== used || Number(row.bytes) !== used) await ctx.write(STORE, storeId, { usage_bytes: used, bytes: used }, 'vector_store.update');
+}
+
+/** A store as a read finds it: the vendor's indexing of its files observed first, and its size with them. */
+async function observeStore(ctx: SemanticsContext, storeId: string): Promise<void> {
+  for (const b of new Set(attached(ctx, storeId).map((r) => r._batch_id).filter((x) => x !== undefined))) await landCancel(ctx, String(b));
+  await progressAll(ctx, FILE, indexed(ctx), (r) => r.vector_store_id === storeId && !cancelPending(ctx, r._batch_id));
+  await recount(ctx, storeId);
+}
+
+/** The store the path names (a deleted store still holds its files), or OpenAI's answer that there is none. */
+function store(ctx: SemanticsContext): { id: string } | { answer: Response } {
+  const id = at(ctx, 'vector_store_id');
+  return ctx.row(STORE, id, { withDeleted: true }) ? { id } : { answer: ctx.notFound(STORE, id) };
+}
+
+/** The store's attached files, bookkeeping included (which batch attached each). */
+const attached = (ctx: SemanticsContext, storeId: string): Row[] => ctx.rowsRaw(FILE).filter((r) => r.vector_store_id === storeId);
+
+/** A finished file batch counts its files done. */
+const batchDone = (row: Row, end: string): Row => {
+  const total = Number((row.file_counts as { total?: unknown } | undefined)?.total ?? 0);
+  return end === 'completed' ? { file_counts: { in_progress: 0, completed: total, failed: 0, cancelled: 0, total } } : {};
+};
+
+/** Observe the vendor's work on a file batch and the files it attached. */
+async function workBatch(ctx: SemanticsContext, batchId: string): Promise<void> {
+  // a batch being cancelled does not finish: it stops once its cancel lands
+  if (await landCancel(ctx, batchId)) return;
+  await progress(ctx, BATCH, batchId, batchDone);
+  await progressAll(ctx, FILE, indexed(ctx), (r) => r._batch_id === batchId);
+  const b = ctx.row(BATCH, batchId);
+  if (b) await recount(ctx, String(b.vector_store_id));
+}
+
+const create: Semantics = async (ctx) => {
+  const p = ctx.params;
+  const id = ctx.mint(STORE);
+  const created = epoch(ctx);
+  const fileIds = Array.isArray(p.file_ids) ? (p.file_ids as unknown[]) : [];
+  const fields = {
+    object: 'vector_store',
+    created_at: created,
+    name: typeof p.name === 'string' ? p.name : null,
+    // what it is for, when the request says, and its size, as the reference's create example answers them
+    // (spec/patches.json adds them to the store object)
+    description: typeof p.description === 'string' ? p.description : null,
+    bytes: 0,
+    usage_bytes: 0,
+    status: 'completed',
+    file_counts: { in_progress: 0, completed: fileIds.length, failed: 0, cancelled: 0, total: fileIds.length },
+    metadata: objectOr(p.metadata, {}),
+    last_active_at: created,
+  };
+  const body = await ctx.write(STORE, id, fields, 'vector_store.create');
+  for (const fid of fileIds) await attach(ctx, id, String(fid));
+  return ctx.reply(body);
+};
+
+const createFile: Semantics = async (ctx) => {
+  const found = store(ctx);
+  if ('answer' in found) return found.answer;
+  if (ctx.params.file_id === undefined || ctx.params.file_id === '') return invalid(ctx, 'you must provide a file_id parameter', 'file_id');
+  return ctx.reply(await attach(ctx, found.id, String(ctx.params.file_id), undefined, ctx.params.attributes));
+};
+
+const listFiles: Semantics = async (ctx) => {
+  const found = store(ctx);
+  if ('answer' in found) return found.answer;
+  for (const b of new Set(attached(ctx, found.id).map((r) => r._batch_id).filter((x) => x !== undefined))) await landCancel(ctx, String(b));
+  await progressAll(ctx, FILE, indexed(ctx), (r) => r.vector_store_id === found.id && !cancelPending(ctx, r._batch_id));
+  await recount(ctx, found.id);
+  return page(ctx, attached(ctx, found.id).map((r) => ctx.get(FILE, String(r.id))!));
+};
+
+/** The attached file the path names, or OpenAI's answer naming both ids. */
+function attachedFile(ctx: SemanticsContext, storeId: string): { key: string; file: Row } | { answer: Response } {
+  const fileId = at(ctx, 'file_id');
+  const key = `${storeId}::${fileId}`;
+  const file = ctx.get(FILE, key);
+  return file ? { key, file } : { answer: missing(ctx, `No file found with id '${fileId}' in vector store '${storeId}'.`) };
+}
+
+const getFile: Semantics = async (ctx) => {
+  const found = store(ctx);
+  if ('answer' in found) return found.answer;
+  const key = `${found.id}::${at(ctx, 'file_id')}`;
+  const batchOf = ctx.row(FILE, key)?._batch_id;
+  if (batchOf !== undefined) await landCancel(ctx, String(batchOf));
+  if (!cancelPending(ctx, batchOf)) await progress(ctx, FILE, key, indexed(ctx));
+  await recount(ctx, found.id);
+  const hit = attachedFile(ctx, found.id);
+  return 'answer' in hit ? hit.answer : ctx.reply(hit.file);
+};
+
+// detaching a file leaves the File itself in place
+const deleteFile: Semantics = async (ctx) => {
+  const found = store(ctx);
+  if ('answer' in found) return found.answer;
+  const hit = attachedFile(ctx, found.id);
+  if ('answer' in hit) return hit.answer;
+  await ctx.write(FILE, hit.key, { deleted: true }, 'vector_store_file.delete');
+  await recount(ctx, found.id);
+  return ctx.reply({ id: hit.file.id, object: 'vector_store.file.deleted', deleted: true });
+};
+
+// A file batch attaches each file, then records the batch, in progress until a read looks.
+const createBatch: Semantics = async (ctx) => {
+  const found = store(ctx);
+  if ('answer' in found) return found.answer;
+  // the files as ids (`file_ids`) or each with its own attributes (`files`), one or the other
+  // (https://platform.openai.com/docs/api-reference/vector-stores-file-batches/createBatch)
+  const ids = ctx.params.file_ids;
+  const files = ctx.params.files;
+  const entries: Row[] | undefined = Array.isArray(files) ? (files as Row[]) : Array.isArray(ids) ? ids.map((file_id) => ({ file_id })) : undefined;
+  if (!entries || entries.length === 0 || (Array.isArray(files) && Array.isArray(ids))) return invalid(ctx, 'you must provide a non-empty file_ids or files array, not both', Array.isArray(files) ? 'files' : 'file_ids');
+  const fileIds = entries.map((e) => String(e.file_id));
+  const batchId = ctx.mint(BATCH);
+  for (const e of entries) await attach(ctx, found.id, String(e.file_id), batchId, e.attributes);
+  const fields = {
+    object: 'vector_store.files_batch',
+    vector_store_id: found.id,
+    created_at: epoch(ctx),
+    status: 'in_progress',
+    file_counts: { in_progress: fileIds.length, completed: 0, failed: 0, cancelled: 0, total: fileIds.length },
+    _file_ids: fileIds,
+  };
+  return ctx.reply(await ctx.write(BATCH, batchId, fields, 'vector_store_file_batch.create'));
+};
+
+const listBatchFiles: Semantics = async (ctx) => {
+  const found = store(ctx);
+  if ('answer' in found) return found.answer;
+  const batchId = at(ctx, 'batch_id');
+  if (!ctx.row(BATCH, batchId)) return ctx.notFound(BATCH, batchId);
+  await workBatch(ctx, batchId);
+  return page(ctx, attached(ctx, found.id).filter((r) => r._batch_id === batchId).map((r) => ctx.get(FILE, String(r.id))!));
+};
+
+/** A deterministic pseudo-score for (query, file): the twin embeds nothing, so it cannot rank by meaning. */
+function score(query: string, fileId: string): number {
+  let h = 0x811c9dc5;
+  const s = query + '|' + fileId;
+  for (let i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 0x01000193); }
+  return (h >>> 0) / 0xffffffff;
+}
+
+/** A comparison filter's operators (the spec's ComparisonFilter `type`). */
+const COMPARE: Record<string, (v: unknown, want: unknown) => boolean> = {
+  eq: (v, w) => v === w, ne: (v, w) => v !== w,
+  gt: (v, w) => v !== undefined && (v as number) > (w as number), gte: (v, w) => v !== undefined && (v as number) >= (w as number),
+  lt: (v, w) => v !== undefined && (v as number) < (w as number), lte: (v, w) => v !== undefined && (v as number) <= (w as number),
+  in: (v, w) => Array.isArray(w) && w.includes(v), nin: (v, w) => Array.isArray(w) && !w.includes(v),
+};
+
+/** Whether a file's attributes pass a search filter: a comparison of one attribute to a value, or an `and`/`or` of
+ *  filters (the spec's ComparisonFilter and CompoundFilter); no filter passes every file. */
+function passes(filter: unknown, attributes: Row): boolean {
+  if (!filter || typeof filter !== 'object') return true;
+  const f = filter as { type?: unknown; key?: unknown; value?: unknown; filters?: unknown };
+  const all = Array.isArray(f.filters) ? f.filters : [];
+  return f.type === 'and' ? all.every((x) => passes(x, attributes)) : f.type === 'or' ? all.some((x) => passes(x, attributes)) : (COMPARE[String(f.type ?? 'eq')]?.(attributes[String(f.key)], f.value) ?? false);
+}
+
+// the store's files that pass the filter, ranked by the pseudo-score, in OpenAI's search-results page, each named by
+// its File's filename and carrying its attributes (https://platform.openai.com/docs/api-reference/vector-stores/search)
+const search: Semantics = async (ctx) => {
+  const found = store(ctx);
+  if ('answer' in found) return found.answer;
+  const p = ctx.params;
+  if (p.query === undefined || p.query === '') return invalid(ctx, 'you must provide a query parameter', 'query');
+  const query = typeof p.query === 'string' ? p.query : JSON.stringify(p.query);
+  const maxResults = Number.isInteger(Number(p.max_num_results)) ? Number(p.max_num_results) : 10;
+  const data = attached(ctx, found.id)
+    .filter((f) => passes(p.filters, (f.attributes ?? {}) as Row))
+    .map((f) => ({ file_id: String(f.file_id ?? ''), attributes: (f.attributes ?? {}) as Row, score: score(query, String(f.file_id ?? '')) }))
+    .sort((a, b) => b.score - a.score)
+    .slice(0, Math.max(1, maxResults))
+    .map((r) => ({ file_id: r.file_id, filename: String(ctx.row('OpenAIFile', r.file_id, { withDeleted: true })?.filename ?? r.file_id), score: r.score, attributes: r.attributes, content: [{ type: 'text', text: `[twin-stub] deterministic pseudo-match for "${query}" in ${r.file_id} (no real semantic search)` }] }));
+  return ctx.reply({ object: 'vector_store.search_results.page', search_query: query, data, has_more: false, next_page: null });
+};
+
+// Cancelling a batch "attempts to cancel the processing of files in this batch as soon as possible"
+// (https://platform.openai.com/docs/api-reference/vector-stores-file-batches/cancelBatch): the cancel answers the batch as
+// it stands, still `in_progress`, and the batch stops once the World clock passes the work already in flight, each of its
+// files still in progress cancelled; what it finished stays finished. Extrapolation, the twin's own timing: the work in
+// flight takes a second.
+const CANCEL_SECONDS = 1;
+
+/** Whether a batch has a cancel asked for that has not landed yet. */
+const cancelPending = (ctx: SemanticsContext, batchId: unknown): boolean => batchId !== undefined && ctx.row(BATCH, String(batchId))?._cancel_asked_at !== undefined && ctx.row(BATCH, String(batchId))?.status === 'in_progress';
+
+/** A batch whose cancel is due stops: its files still in progress cancelled, its counts with them. */
+async function landCancel(ctx: SemanticsContext, batchId: string): Promise<boolean> {
+  const batch = ctx.row(BATCH, batchId);
+  if (!batch || batch.status !== 'in_progress' || batch._cancel_asked_at === undefined || batch._cancel_asked_at === null) return false;
+  if (epoch(ctx) < Number(batch._cancel_asked_at) + CANCEL_SECONDS) return true;
+  // the vendor's move, not the caller's: the cancel lands when a read looks
+  if (ctx.legal(BATCH, 'status', ctx.call.operation.id, batch.status, 'cancelled', batchId, 'vendor')) return true;
+  const stopped = attached(ctx, String(batch.vector_store_id)).filter((r) => r._batch_id === batchId && r.status === 'in_progress');
+  for (const f of stopped) {
+    ctx.legal(FILE, 'status', ctx.call.operation.id, f.status, 'cancelled', String(f.id), 'vendor');
+    await ctx.write(FILE, String(f.id), { status: 'cancelled' }, 'vector_store_file.cancel');
+  }
+  const total = Number((batch.file_counts as { total?: unknown } | undefined)?.total ?? 0);
+  const file_counts = { in_progress: 0, completed: total - stopped.length, failed: 0, cancelled: stopped.length, total };
+  await ctx.write(BATCH, batchId, { status: 'cancelled', file_counts }, 'vector_store_file_batch.cancel');
+  return true;
+}
+
+const cancelBatch: Semantics = async (ctx) => {
+  const found = store(ctx);
+  if ('answer' in found) return found.answer;
+  // a cancel is not a read: it finds the batch where the vendor's work has got to without looking
+  const batchId = at(ctx, 'batch_id');
+  const batch = ctx.row(BATCH, batchId);
+  if (!batch || batch.vector_store_id !== found.id) return ctx.notFound(BATCH, batchId);
+  const refusal = ctx.legal(BATCH, 'status', 'cancelVectorStoreFileBatch', batch.status, 'cancelled');
+  if (refusal) return ctx.refuse(refusal);
+  if (batch._cancel_asked_at !== undefined && batch._cancel_asked_at !== null) return ctx.reply(ctx.get(BATCH, batchId)!);
+  return ctx.reply(await ctx.write(BATCH, batchId, { _cancel_asked_at: epoch(ctx) }, 'vector_store_file_batch.update'));
+};
+
+export const vectorStores: Record<string, Semantics> = {
+  getVectorStore: async (ctx) => {
+    if (!readOnly(ctx)) await observeStore(ctx, at(ctx, 'vector_store_id'));
+    return coreAnswer(ctx);
+  },
+  listVectorStores: async (ctx) => {
+    if (!readOnly(ctx)) for (const r of ctx.rowsRaw(STORE)) await observeStore(ctx, String(r.id));
+    return coreAnswer(ctx);
+  },
+  cancelVectorStoreFileBatch: cancelBatch,
+  createVectorStore: create,
+  createVectorStoreFile: createFile,
+  listVectorStoreFiles: listFiles,
+  getVectorStoreFile: getFile,
+  getVectorStoreFileBatch: async (ctx) => {
+    await workBatch(ctx, at(ctx, 'batch_id'));
+    return coreAnswer(ctx);
+  },
+  deleteVectorStoreFile: deleteFile,
+  createVectorStoreFileBatch: createBatch,
+  listFilesInVectorStoreBatch: listBatchFiles,
+  searchVectorStore: search,
+};

package/test-fixtures/openai-openapi-operations.SOURCE.md

@@ -1,19 +1,18 @@
 # openai-openapi-operations.json — provenance
 
 The operation-inventory denominator for `../openai-spec-census.json`: a scope-filtered subset of
-the vendor's published spec, enumerated by `deriveFromOpenAPI` (`packages/twin/tooling/src/derive.ts`).
+the vendor's published spec, enumerated by `deriveFromOpenAPI` (`packages/world-tooling/src/derive.ts`).
 
 - **Upstream:** https://github.com/openai/openai-openapi — `openapi.yaml`, fetched 2026-09-02 from
   `https://raw.githubusercontent.com/openai/openai-openapi/master/openapi.yaml` at commit
   `18a43ed13461a01fce5b9cc93c86843c550734a3` (2026-09-01), spec `info.version` 2.3.0.
 - **How it was built:** `bun scripts/spec-census-bootstrap.ts --vendor openai --from-file <openapi.yaml> --scope-paths /assistants,/audio,/batches,/chat/,/containers,/embeddings,/evals,/files,/fine_tuning,/images,/models,/moderations,/organization,/responses,/threads,/uploads,/vector_stores --apply`
-  over a copy of the upstream file fetched 2026-09-02; operation metadata only (operationId, summary, tags,
-  first response), schemas dropped. The gate never fetches — refreshing the fixture is
+  over a copy of the upstream file fetched 2026-09-02; operation metadata only (operationId, summary, tags), schemas and response codes dropped. The gate never fetches — refreshing the fixture is
   `bun scripts/spec-refresh.ts --vendor openai --from-file <newer spec> --apply` over a newly fetched copy.
 - **Scope (222 of 281 upstream operations):** every operation whose path starts with one of
   `/assistants`, `/audio`, `/batches`, `/chat/`, `/containers`, `/embeddings`, `/evals`, `/files`, `/fine_tuning`, `/images`, `/models`, `/moderations`, `/organization`, `/responses`, `/threads`, `/uploads`, `/vector_stores`.
 - **Mapping:** 128 operations mapped to capability ids by segment match at birth, then reviewed by
   hand against the router (`routeOpenAI` in `../src/openai-twin.ts`) and each capability's `verify()`:
-  60 entries were re-ruled, landing at **97 capability / 0 outOfScope / 125 unmapped** (the ratchet
-  baseline). Every later ruling (an `outOfScope` reason, a remap) is an edit to the census, never to
+  60 entries were re-ruled, landing at **97 capability / 0 $ruled / 125 unmapped** (the ratchet
+  baseline). Every later ruling (a `$ruled` marker, a remap) is an edit to the census, never to
   this fixture.