@opencraw/azure-durable 0.0.2

package/dist/src/result-store/result-packing.policy.d.ts

@@ -0,0 +1,22 @@
+import type { OutputRecord } from '@opencraw/core';
+import type { ResultStore } from './result-store.contract.js';
+/** Records returned in the Durable output, or where to read them when they were too large. */
+export interface PackedRecords {
+    records?: OutputRecord[];
+    resultUrl?: string;
+}
+/**
+ * Returns records inline when small, else saves them and returns a link: an
+ * orchestration's history holds every activity output, so a large result
+ * belongs in storage, not in the task hub.
+ *
+ * @param records - The records.
+ * @param options - The store (without one, records are always inline), the limit, the name to save under.
+ * @returns The records or their link.
+ */
+export declare function packRecords(records: readonly OutputRecord[], options: {
+    store?: ResultStore;
+    inlineLimitBytes: number;
+    name: string;
+}): Promise<PackedRecords>;
+//# sourceMappingURL=result-packing.policy.d.ts.map

package/dist/src/result-store/result-store.contract.d.ts

@@ -0,0 +1,13 @@
+import type { OutputRecord } from '@opencraw/core';
+/** Where results too large to return inline go. */
+export interface ResultStore {
+    /**
+     * Saves records as JSON Lines.
+     *
+     * @param name - A path-like name, unique per result: `<instance>/<recipe>.jsonl`.
+     * @param records - The records.
+     * @returns A URL the caller can read them from.
+     */
+    save(name: string, records: readonly OutputRecord[]): Promise<string>;
+}
+//# sourceMappingURL=result-store.contract.d.ts.map

package/dist/src/worker-pools/index.d.ts

@@ -0,0 +1,10 @@
+export { WorkerPools } from './worker-pools.store.js';
+export { itemOrchestrator } from './item-orchestration.use-case.js';
+export { submitItem } from './submit-item.handler.js';
+export { jobRequest } from './job.handler.js';
+export { capWindows } from './windows-cap.policy.js';
+export { itemRequestSchema, crawlIdSchema } from './item-request.validator.js';
+export type { ItemRequest } from './item-request.validator.js';
+export type { ItemJob, ItemOutcome, PoolState } from './pool-job.model.js';
+export { parseItemJob } from './item-job.validator.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/worker-pools/item-job.validator.d.ts

@@ -0,0 +1,10 @@
+import type { ItemJob } from './pool-job.model.js';
+/**
+ * Checks a pooled item's orchestration input as it comes back out of the task hub.
+ *
+ * @param raw - The stored input.
+ * @returns The job.
+ * @throws ZodError when it is not one.
+ */
+export declare function parseItemJob(raw: unknown): ItemJob;
+//# sourceMappingURL=item-job.validator.d.ts.map

package/dist/src/worker-pools/item-orchestration.use-case.d.ts

@@ -0,0 +1,14 @@
+import type { TypedActivity } from '@mnci/az-durable';
+import type { OrchestrationContext, Task } from 'durable-functions';
+import type { ItemJob, ItemOutcome } from './pool-job.model.js';
+/**
+ * A pooled item's orchestration: one activity, which hands the item to the
+ * warm pool and waits for its result. Durable adds what a plain HTTP call
+ * lacks: a status URL to poll past the HTTP time limit, and an instance id
+ * that refuses a duplicate while the item runs.
+ *
+ * @param runItem - The activity.
+ * @returns The orchestrator body.
+ */
+export declare function itemOrchestrator(runItem: TypedActivity<ItemJob, ItemOutcome>): (context: OrchestrationContext, job: ItemJob) => Generator<Task, ItemOutcome, unknown>;
+//# sourceMappingURL=item-orchestration.use-case.d.ts.map

package/dist/src/worker-pools/item-request.validator.d.ts

@@ -0,0 +1,35 @@
+import { z } from 'zod';
+export declare const crawlIdSchema: z.ZodString;
+/** A `/jobs/{crawlId}/items` body. */
+export declare const itemRequestSchema: z.ZodObject<{
+    recipe: z.ZodObject<{
+        name: z.ZodString;
+        version: z.ZodString;
+    }, z.core.$strict>;
+    item: z.ZodObject<{
+        id: z.ZodString;
+        vars: z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+        recipe: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
+    jobSize: z.ZodOptional<z.ZodNumber>;
+    windows: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodObject<{
+        min: z.ZodOptional<z.ZodNumber>;
+        max: z.ZodOptional<z.ZodNumber>;
+        start: z.ZodOptional<z.ZodNumber>;
+        grow: z.ZodOptional<z.ZodObject<{
+            after: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strict>>;
+        shrink: z.ZodOptional<z.ZodEnum<{
+            one: "one";
+            half: "half";
+        }>>;
+        restart: z.ZodOptional<z.ZodObject<{
+            after: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strict>>;
+        idle: z.ZodOptional<z.ZodObject<{
+            afterMs: z.ZodNumber;
+        }, z.core.$strict>>;
+    }, z.core.$strict>]>>;
+}, z.core.$strict>;
+export type ItemRequest = z.infer<typeof itemRequestSchema>;
+//# sourceMappingURL=item-request.validator.d.ts.map

package/dist/src/worker-pools/job.handler.d.ts

@@ -0,0 +1,14 @@
+import type { HttpRequest, HttpResponseInit } from '@azure/functions';
+import type { HostSettings } from '../host-options/index.js';
+import type { WorkerPools } from './worker-pools.store.js';
+/**
+ * `GET /jobs/{crawlId}`: the pool's state. `DELETE /jobs/{crawlId}`: closes
+ * it; queued and running items finish first.
+ *
+ * @param request - The request.
+ * @param settings - The host settings.
+ * @param pools - This process's pools.
+ * @returns The state, 202 when closing, 404 when there is no pool.
+ */
+export declare function jobRequest(request: HttpRequest, settings: HostSettings, pools: WorkerPools): Promise<HttpResponseInit>;
+//# sourceMappingURL=job.handler.d.ts.map

package/dist/src/worker-pools/pool-job.model.d.ts

@@ -0,0 +1,43 @@
+import type { OutputRecord, RecipeReport, WindowsPolicy, WorkItem, WorkOutcome } from '@opencraw/core';
+/** What a pooled item's orchestration runs: written by the starter, never by the caller. */
+export interface ItemJob {
+    /** The caller and crawl id: one pool per key. */
+    key: string;
+    crawlId: string;
+    allowedHosts: string[];
+    recipe: {
+        name: string;
+        version: string;
+    };
+    item: WorkItem;
+    jobSize?: number;
+    windows?: WindowsPolicy | number;
+    /** Where the item's records go when too large to return. */
+    resultName: string;
+}
+/** A pool as a caller sees it. */
+export interface PoolState {
+    crawlId: string;
+    recipe: {
+        name: string;
+        version: string;
+    };
+    /** The windows the pool aims for. */
+    windows: number;
+    /** Windows waiting for an item: while there are any, the caller could send more. */
+    idle: number;
+    queued: number;
+    running: number;
+    ended: Record<WorkOutcome, number>;
+    closing: boolean;
+}
+/** How a pooled item ended. `refused`: the pool would not take it (another recipe version, a missing or draft recipe set). */
+export interface ItemOutcome {
+    outcome: WorkOutcome | 'refused';
+    report?: RecipeReport;
+    records?: OutputRecord[];
+    resultUrl?: string;
+    error?: string;
+    pool?: PoolState;
+}
+//# sourceMappingURL=pool-job.model.d.ts.map

package/dist/src/worker-pools/submit-item.handler.d.ts

@@ -0,0 +1,20 @@
+import type { HttpRequest, HttpResponse, HttpResponseInit } from '@azure/functions';
+import type { TypedOrchestration } from '@mnci/az-durable';
+import type { DurableClient } from 'durable-functions';
+import type { HostSettings } from '../host-options/index.js';
+import type { ItemJob, ItemOutcome } from './pool-job.model.js';
+import type { WorkerPools } from './worker-pools.store.js';
+/**
+ * `POST /jobs/{crawlId}/items`: one item for the caller's warm pool. The
+ * Durable instance id is the caller, crawl id and item id, so the same item
+ * sent again while it runs returns the running instance's status.
+ *
+ * @param request - The request.
+ * @param client - The Durable client.
+ * @param settings - The host settings.
+ * @param pools - This process's pools.
+ * @param orchestration - The item orchestration.
+ * @returns 202 with the Durable status URLs, or the problem.
+ */
+export declare function submitItem(request: HttpRequest, client: DurableClient, settings: HostSettings, pools: WorkerPools, orchestration: TypedOrchestration<ItemJob, ItemOutcome>): Promise<HttpResponse | HttpResponseInit>;
+//# sourceMappingURL=submit-item.handler.d.ts.map

package/dist/src/worker-pools/windows-cap.policy.d.ts

@@ -0,0 +1,12 @@
+import type { WindowsPolicy } from '@opencraw/core';
+/**
+ * A job's windows, within the host's limit and never more than the job has
+ * items: twelve windows for three items would only sit idle.
+ *
+ * @param policy - What the job asked for.
+ * @param maxWindows - The host's limit.
+ * @param jobSize - About how many items the job has.
+ * @returns The policy, capped.
+ */
+export declare function capWindows(policy: WindowsPolicy | number, maxWindows: number, jobSize?: number): WindowsPolicy;
+//# sourceMappingURL=windows-cap.policy.d.ts.map

package/dist/src/worker-pools/worker-pools.store.d.ts

@@ -0,0 +1,58 @@
+import type { HostSettings } from '../host-options/index.js';
+import type { ItemJob, ItemOutcome, PoolState } from './pool-job.model.js';
+/**
+ * The warm worker pools of this process, one per caller and crawl id. A pool
+ * is created by its first item, runs `crawler.work` on an inbox, and closes
+ * when its job size is reached, when it has had no item for `idleTtlMs`, or
+ * when asked. Pools live in this process's memory: a Function App scaled to
+ * several instances has one pool per instance for the same crawl id.
+ */
+export declare class WorkerPools {
+    private readonly settings;
+    private readonly now;
+    private readonly pools;
+    /** Each pool's recipe version, known from the moment it starts opening. */
+    private readonly versions;
+    private readonly live;
+    private sweeper;
+    constructor(settings: HostSettings, now?: () => number);
+    private open;
+    private create;
+    /** Waits for a pool to run dry, then closes its browser and forgets it; a pool that failed rejects what it still held. */
+    private work;
+    private forget;
+    /** Starts looking for pools that have had no item for `idleTtlMs`, while any is open. */
+    private sweep;
+    /**
+     * The recipe version a pool runs, if one is open under this key.
+     *
+     * @param key - The caller and crawl id.
+     * @returns The version.
+     */
+    versionOf(key: string): {
+        name: string;
+        version: string;
+    } | undefined;
+    /**
+     * @param key - The caller and crawl id.
+     * @returns The pool as a caller sees it, if one is open.
+     */
+    state(key: string): PoolState | undefined;
+    /**
+     * Runs an item on its job's pool, opening the pool when it is the first.
+     *
+     * @param job - The item and its job.
+     * @returns How it ended, with the pool's state.
+     */
+    run(job: ItemJob): Promise<ItemOutcome>;
+    /**
+     * Closes a pool: queued and running items finish, then its browser closes.
+     *
+     * @param key - The caller and crawl id.
+     * @returns Whether there was a pool to close.
+     */
+    close(key: string): Promise<boolean>;
+    /** Closes every pool, as the host shuts down. */
+    closeAll(): Promise<void>;
+}
+//# sourceMappingURL=worker-pools.store.d.ts.map

package/package.json

@@ -0,0 +1,93 @@
+{
+  "name": "@opencraw/azure-durable",
+  "version": "0.0.2",
+  "type": "module",
+  "main": "./dist/index.esm.js",
+  "module": "./dist/index.esm.js",
+  "types": "./dist/src/index.d.ts",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/src/index.d.ts",
+      "import": "./dist/index.esm.js",
+      "default": "./dist/index.esm.js"
+    }
+  },
+  "files": [
+    "dist",
+    "!**/*.tsbuildinfo",
+    "!**/*.d.ts.map",
+    "!**/*.js.map"
+  ],
+  "dependencies": {
+    "@azure/functions": ">=4.0.0 <5",
+    "@azure/storage-blob": "^12.34.0",
+    "@mnci/az-durable": "^0.1.12",
+    "@modelcontextprotocol/sdk": "^1.30.1",
+    "@opencraw/cli": "^0.1.9",
+    "@opencraw/core": "^0.1.9",
+    "@opencraw/mcp": "^0.1.9",
+    "durable-functions": ">=3.0.0 <4",
+    "zod": "^4.6.5"
+  },
+  "peerDependencies": {
+    "@azure/functions": ">=4.0.0 <5",
+    "durable-functions": ">=3.0.0 <4"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "description": "OpenCraw over HTTP on Azure Durable Functions: run recipes with one call, or feed warm worker pools one item at a time.",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/russoedu/open.craw.git",
+    "directory": "packages/azure-durable"
+  },
+  "keywords": [
+    "crawler",
+    "scraper",
+    "azure-functions",
+    "durable-functions",
+    "recipe"
+  ],
+  "engines": {
+    "node": ">=22"
+  },
+  "sideEffects": false,
+  "nx": {
+    "targets": {
+      "typecheck": {
+        "dependsOn": [
+          "^build"
+        ]
+      },
+      "lint": {
+        "dependsOn": [
+          "^build"
+        ]
+      },
+      "e2e": {
+        "executor": "nx:run-commands",
+        "dependsOn": [
+          "build",
+          {
+            "projects": [
+              "@opencraw/core"
+            ],
+            "target": "build"
+          }
+        ],
+        "inputs": [
+          "default",
+          "^production",
+          "{workspaceRoot}/packages/core/e2e/**"
+        ],
+        "options": {
+          "command": "tsc -p tsconfig.e2e.json && jest --config jest.e2e.config.cts",
+          "cwd": "packages/azure-durable"
+        }
+      }
+    }
+  }
+}