@anyformat/sdk 1.10.0 → 1.12.0

package/README.md

@@ -103,6 +103,33 @@ await workflow.uploadDocumentsToDataset([
 Both are also available off the client: `af.uploadToDataset(workflowId, files, opts)`
 and `af.uploadDocumentsToDataset(workflowId, documents)`.
 
+## Evals
+
+An eval re-extracts the dataset, or a slice of it picked by dataset tags, on one
+workflow version and grades it against ground truth. It is billed per document,
+so pass an `idempotencyKey` and an explicit `versionId` to make a retry replay
+the original eval:
+
+```ts
+const q3 = (await af.listDatasetTags(workflowId)).filter((t) => t.name === "Q3 invoices").map((t) => t.id);
+const current = (await af.listWorkflowVersions(workflowId, { limit: 1 })).items[0]!.version_id;
+const { eval_id } = await af.launchEval(workflowId, {
+  versionId: current,
+  datasetTagIds: q3,
+  untagged: true, // also the documents with no tag
+  idempotencyKey: `eval-${new Date().toISOString().slice(0, 10)}`, // one eval per day
+});
+
+let detail = await af.getEval(workflowId, eval_id); // poll until status leaves "in_progress"
+const accuracy = await af.getEvalAccuracy(workflowId, eval_id); // per field, document, tag
+```
+
+`getEvalDocumentPacketFields` shows one document's extracted values beside its
+ground truth. To compare versions, read the evals `listEvals` returns: each
+carries its `version_id`. To compare tags, read `by_dataset_tag` in the eval's
+accuracy. Accuracy rows name fields by `persistent_id`;
+`getWorkflow(workflowId, { version })` gives the names.
+
 ## Idempotency keys
 
 Upload and run triggers accept an `Idempotency-Key` (Stripe convention) —

package/dist/index.cjs

@@ -52,7 +52,7 @@ module.exports = __toCommonJS(index_exports);
 // package.json
 var package_default = {
   name: "@anyformat/sdk",
-  version: "1.10.0",
+  version: "1.12.0",
   description: "TypeScript SDK for the AnyFormat API",
   license: "MIT",
   author: "Anyformat <info@anyformat.ai>",
@@ -1178,6 +1178,11 @@ var v3UpdateWorkflow = (options) => (options.client ?? client).patch({
     ...options.headers
   }
 });
+var v3ListDatasetTags = (options) => (options.client ?? client).get({
+  security: [{ scheme: "bearer", type: "http" }],
+  url: "/v3/workflows/{workflow_id}/dataset/tags/",
+  ...options
+});
 var v3UploadToDataset = (options) => (options.client ?? client).post({
   ...formDataBodySerializer,
   security: [{ scheme: "bearer", type: "http" }],
@@ -1193,6 +1198,35 @@ var v3ListWorkflowDocumentPackets = (options) => (options.client ?? client).get(
   url: "/v3/workflows/{workflow_id}/document-packets/",
   ...options
 });
+var v3ListEvals = (options) => (options.client ?? client).get({
+  security: [{ scheme: "bearer", type: "http" }],
+  url: "/v3/workflows/{workflow_id}/evals/",
+  ...options
+});
+var v3LaunchEval = (options) => (options.client ?? client).post({
+  security: [{ scheme: "bearer", type: "http" }],
+  url: "/v3/workflows/{workflow_id}/evals/",
+  ...options,
+  headers: {
+    "Content-Type": "application/json",
+    ...options.headers
+  }
+});
+var v3GetEval = (options) => (options.client ?? client).get({
+  security: [{ scheme: "bearer", type: "http" }],
+  url: "/v3/workflows/{workflow_id}/evals/{eval_id}/",
+  ...options
+});
+var v3GetEvalAccuracy = (options) => (options.client ?? client).get({
+  security: [{ scheme: "bearer", type: "http" }],
+  url: "/v3/workflows/{workflow_id}/evals/{eval_id}/accuracy/",
+  ...options
+});
+var v3GetEvalDocumentPacketFields = (options) => (options.client ?? client).get({
+  security: [{ scheme: "bearer", type: "http" }],
+  url: "/v3/workflows/{workflow_id}/evals/{eval_id}/document-packets/{document_packet_id}/fields/",
+  ...options
+});
 var v3ListWorkflowRuns = (options) => (options.client ?? client).get({
   security: [{ scheme: "bearer", type: "http" }],
   url: "/v3/workflows/{workflow_id}/runs/",
@@ -1227,6 +1261,11 @@ var v3UploadAndRunWorkflow = (options) => (options.client ?? client).post({
     ...options.headers
   }
 });
+var v3ListWorkflowVersions = (options) => (options.client ?? client).get({
+  security: [{ scheme: "bearer", type: "http" }],
+  url: "/v3/workflows/{workflow_id}/versions/",
+  ...options
+});
 
 // src/run.ts
 function isTransientStatus(status) {
@@ -1665,9 +1704,16 @@ var Anyformat = class {
       nextCursor: res.data.next_cursor ?? null
     };
   }
-  /** Retrieve a workflow with its complete typed graph inline (round-trippable into `updateWorkflow`). */
-  async getWorkflow(workflowId) {
-    const res = await v3GetWorkflow({ client: this._client, path: { workflow_id: workflowId } });
+  /**
+   * Retrieve a workflow with its complete typed graph inline (round-trippable into `updateWorkflow`).
+   * Pass a `version_id` as `version` to read that version instead of the current one.
+   */
+  async getWorkflow(workflowId, opts = {}) {
+    const res = await v3GetWorkflow({
+      client: this._client,
+      path: { workflow_id: workflowId },
+      ...opts.version ? { query: { version: opts.version } } : {}
+    });
     if (res.error) throw toApiError(res.response?.status ?? 500, res.error, res.response?.headers);
     return res.data;
   }
@@ -1791,6 +1837,26 @@ var Anyformat = class {
       cursor = page.nextCursor;
     }
   }
+  /** A workflow's versions, newest first; item 0 of the first page is the current version. */
+  async listWorkflowVersions(workflowId, opts = {}) {
+    const res = await v3ListWorkflowVersions({
+      client: this._client,
+      path: { workflow_id: workflowId },
+      query: { limit: opts.limit, cursor: opts.cursor }
+    });
+    if (res.error) throw toApiError(res.response?.status ?? 500, res.error, res.response?.headers);
+    return { items: res.data.items, nextCursor: res.data.next_cursor ?? null };
+  }
+  /** Every version of a workflow, newest first, following `nextCursor` across pages. Lazy. */
+  async *iterWorkflowVersions(workflowId, opts = {}) {
+    let cursor;
+    for (; ; ) {
+      const page = await this.listWorkflowVersions(workflowId, { limit: opts.limit, cursor });
+      yield* page.items;
+      if (page.nextCursor === null) return;
+      cursor = page.nextCursor;
+    }
+  }
   /** Retrieve one run: lifecycle status plus results inline once `processed`. */
   async getRun(runId) {
     const res = await v3GetRun({ client: this._client, path: { run_id: runId } });
@@ -1837,6 +1903,70 @@ var Anyformat = class {
     if (run.status === "cancelled") throw new ExtractionCancelledError(runId);
     return null;
   }
+  /** A workflow's dataset tags, ordered by name; a tag's `id` is what `datasetTagIds` takes. */
+  async listDatasetTags(workflowId) {
+    const res = await v3ListDatasetTags({ client: this._client, path: { workflow_id: workflowId } });
+    if (res.error) throw toApiError(res.response?.status ?? 500, res.error, res.response?.headers);
+    return res.data.items;
+  }
+  /**
+   * Launch an eval: extract every dataset document on a version and grade it
+   * against ground truth. Billed per document. Omit `versionId` for the current
+   * version. `idempotencyKey` makes a retry return the original eval instead of
+   * launching a second one. `datasetTagIds: []` without `untagged: true` throws.
+   */
+  async launchEval(workflowId, opts = {}) {
+    if (opts.datasetTagIds?.length === 0 && !opts.untagged) {
+      throw new AnyformatError(
+        "launchEval() datasetTagIds: [] selects no document; omit datasetTagIds to evaluate the whole dataset, or pass untagged: true"
+      );
+    }
+    const res = await v3LaunchEval({
+      client: this._client,
+      path: { workflow_id: workflowId },
+      body: {
+        ...opts.versionId ? { version_id: opts.versionId } : {},
+        ...opts.datasetTagIds ? { dataset_tag_ids: opts.datasetTagIds } : {},
+        ...opts.untagged ? { untagged: true } : {}
+      },
+      ...opts.idempotencyKey ? { headers: { "Idempotency-Key": opts.idempotencyKey } } : {}
+    });
+    if (res.error) throw toApiError(res.response?.status ?? 500, res.error, res.response?.headers);
+    return res.data;
+  }
+  /** One eval's headline; poll it until `status` leaves `in_progress`. */
+  async getEval(workflowId, evalId) {
+    const res = await v3GetEval({ client: this._client, path: { workflow_id: workflowId, eval_id: evalId } });
+    if (res.error) throw toApiError(res.response?.status ?? 500, res.error, res.response?.headers);
+    return res.data;
+  }
+  async listEvals(workflowId, opts = {}) {
+    const res = await v3ListEvals({
+      client: this._client,
+      path: { workflow_id: workflowId },
+      query: { limit: opts.limit, cursor: opts.cursor }
+    });
+    if (res.error) throw toApiError(res.response?.status ?? 500, res.error, res.response?.headers);
+    return { items: res.data.items, nextCursor: res.data.next_cursor ?? null };
+  }
+  /** An eval's accuracy per field, per document and per dataset tag; partial while it runs. */
+  async getEvalAccuracy(workflowId, evalId) {
+    const res = await v3GetEvalAccuracy({
+      client: this._client,
+      path: { workflow_id: workflowId, eval_id: evalId }
+    });
+    if (res.error) throw toApiError(res.response?.status ?? 500, res.error, res.response?.headers);
+    return res.data;
+  }
+  /** One eval document's extracted values beside its ground truth. The id is the dataset copy's. */
+  async getEvalDocumentPacketFields(workflowId, evalId, documentPacketId) {
+    const res = await v3GetEvalDocumentPacketFields({
+      client: this._client,
+      path: { workflow_id: workflowId, eval_id: evalId, document_packet_id: documentPacketId }
+    });
+    if (res.error) throw toApiError(res.response?.status ?? 500, res.error, res.response?.headers);
+    return res.data;
+  }
 };
 
 // src/schema.ts