@opencraw/azure-durable 0.0.47 → 0.0.49

package/dist/caller-access/caller-access.policy.d.ts

@@ -0,0 +1,28 @@
+import type { HttpRequest } from '@azure/functions';
+/** Who is calling and where their recipes may go. */
+export interface CallerAccess {
+    /** `undefined` when the host does not tell callers apart. */
+    caller?: string;
+    allowedHosts: readonly string[];
+}
+export interface CallerRules {
+    allowedHosts: readonly string[] | ((caller: string | undefined) => readonly string[] | undefined);
+    identify?: (request: HttpRequest) => string | undefined;
+}
+/**
+ * Resolves the caller of a request and the hosts its recipes may reach.
+ *
+ * @param request - The request.
+ * @param rules - The host's `identify` and `allowedHosts`.
+ * @returns The access, or `undefined` when this caller may run nothing.
+ */
+export declare function callerAccess(request: HttpRequest, rules: CallerRules): CallerAccess | undefined;
+/**
+ * The key a caller's pools and jobs live under, so two callers never share a pool.
+ *
+ * @param access - The caller.
+ * @param crawlId - The job.
+ * @returns The key.
+ */
+export declare function poolKey(access: Pick<CallerAccess, 'caller'>, crawlId: string): string;
+//# sourceMappingURL=caller-access.policy.d.ts.map

package/dist/caller-access/index.d.ts

@@ -0,0 +1,3 @@
+export { callerAccess, poolKey } from './caller-access.policy';
+export type { CallerAccess, CallerRules } from './caller-access.policy';
+//# sourceMappingURL=index.d.ts.map

package/dist/callout-resume/callout-event.mapper.d.ts

@@ -0,0 +1,30 @@
+import type { HostSettings } from '../host-options';
+/**
+ * The name of the external event a posted-back result is raised as. The call's idempotency key is the
+ * same for the same call anywhere in the job, so the recipe's index keeps two recipes that make an
+ * identical call apart.
+ *
+ * @param index - The input recipe of the job.
+ * @param key - The call's idempotency key.
+ * @returns The event name.
+ */
+export declare function calloutEventName(index: number, key: string): string;
+/**
+ * Where a handler posts a result, given the URL the job was started at.
+ *
+ * `publicUrl` wins when the host sets it (behind a proxy or a custom domain the request does not show);
+ * otherwise it is the start URL without its last path segment, which is where `/crawl` and
+ * `/callouts/{token}/resolve` are siblings.
+ *
+ * @param startUrl - The URL of the `/crawl` request.
+ * @param settings - The host settings.
+ * @returns The base all callback URLs start with, without a trailing slash.
+ */
+export declare function callbackBaseOf(startUrl: string, settings: Pick<HostSettings, 'callouts'>): string;
+/**
+ * @param base - What `callbackBaseOf` returned.
+ * @param token - The signed ticket.
+ * @returns The URL a handler posts its `CalloutResolution` to.
+ */
+export declare function callbackUrl(base: string, token: string): string;
+//# sourceMappingURL=callout-event.mapper.d.ts.map

package/dist/callout-resume/callout-ticket.algorithm.d.ts

@@ -0,0 +1,36 @@
+/** What a resume token says: which call of which recipe of which job it may settle, and until when. */
+export interface CalloutTicket {
+    instanceId: string;
+    /** The input recipe of the job the call belongs to. */
+    index: number;
+    /** The call's idempotency key. */
+    key: string;
+    /** Epoch milliseconds. */
+    expiresAt: number;
+}
+export type TicketReading = {
+    ok: true;
+    ticket: CalloutTicket;
+} | {
+    ok: false;
+    reason: 'malformed' | 'signature' | 'expired';
+};
+/**
+ * The token a handler posts its result back with. It is the ticket and an HMAC of it, so the host holds no
+ * state for it: any instance of the Function App can check it.
+ *
+ * @param secret - The host's signing key.
+ * @param ticket - What the token lets its holder do.
+ * @returns A URL-safe token.
+ */
+export declare function signCalloutTicket(secret: string, ticket: CalloutTicket): string;
+/**
+ * Checks a token and reads the ticket in it.
+ *
+ * @param secret - The host's signing key.
+ * @param token - What was posted to.
+ * @param now - The current time, epoch milliseconds.
+ * @returns The ticket, or why the token is not good.
+ */
+export declare function readCalloutTicket(secret: string, token: string, now: number): TicketReading;
+//# sourceMappingURL=callout-ticket.algorithm.d.ts.map

package/dist/callout-resume/index.d.ts

@@ -0,0 +1,5 @@
+export { signCalloutTicket, readCalloutTicket } from './callout-ticket.algorithm';
+export type { CalloutTicket, TicketReading } from './callout-ticket.algorithm';
+export { calloutEventName, callbackBaseOf, callbackUrl } from './callout-event.mapper';
+export { resolveCallout } from './resolve-callout.handler';
+//# sourceMappingURL=index.d.ts.map

package/dist/callout-resume/resolve-callout.handler.d.ts

@@ -0,0 +1,17 @@
+import type { HttpRequest, HttpResponseInit } from '@azure/functions';
+import type { DurableClient } from 'durable-functions';
+import type { HostSettings } from '../host-options';
+/**
+ * `POST /callouts/{token}/resolve`: a handler that answered `pending` posts the result of the call here.
+ * The token in the path is the credential (it is a signed ticket for that one call), so the route does
+ * not ask for a function key. The result is raised to the waiting job as an external event; a second post
+ * for a call that was already settled finds nobody waiting and is refused.
+ *
+ * @param request - The request.
+ * @param client - The Durable client.
+ * @param settings - The host settings.
+ * @param now - The current time, epoch milliseconds.
+ * @returns 202 when the result was handed to the job, or the problem.
+ */
+export declare function resolveCallout(request: HttpRequest, client: DurableClient, settings: HostSettings, now?: number): Promise<HttpResponseInit>;
+//# sourceMappingURL=resolve-callout.handler.d.ts.map

package/dist/crawl-run/crawl-job.model.d.ts

@@ -0,0 +1,52 @@
+import type { CalloutResolution, OutputRecord, ParkedCallout, RecipeReport } from '@opencraw/core';
+/** What a `/crawl` orchestration runs: written by the starter, never by the caller. */
+export interface CrawlJob {
+    caller?: string;
+    allowedHosts: string[];
+    output: unknown;
+    inputs: unknown[];
+    dedupe?: 'recipe' | 'off';
+    /** Where handlers post results back (`https://app/api`): set by the starter when the host takes posted-back results. */
+    callbackBase?: string;
+}
+/** One input recipe of a job, run by one activity. */
+export interface RecipeTask {
+    allowedHosts: string[];
+    output: unknown;
+    input: unknown;
+    dedupe?: 'recipe' | 'off';
+    /** Where its records go when too large to return: `<instance>/<index>.jsonl`. */
+    resultName: string;
+    /** The job's instance and this recipe's index in it: what a resume token names. */
+    instanceId: string;
+    index: number;
+    callbackBase?: string;
+    /** Results of calls that parked on an earlier run of this recipe, by idempotency key. */
+    resolutions?: Record<string, CalloutResolution>;
+}
+export interface RecipeTaskResult {
+    /** One per run: a recipe with a `matrix` runs once per variant. */
+    reports: RecipeReport[];
+    records?: OutputRecord[];
+    resultUrl?: string;
+    /** Set when calls are waiting for their results to be posted back: the run stopped, and this is what it waits for. */
+    parked?: ParkedCallout[];
+}
+/** What a `/crawl` orchestration returns. */
+export interface CrawlResult {
+    /** One report per input recipe run (a matrix recipe: per variant), in the order given. */
+    recipes: RecipeReport[];
+    /** The records returned inline. */
+    records: OutputRecord[];
+    /** Links to the records too large to return inline, one per recipe that had them. */
+    results: string[];
+}
+/** The orchestration's custom status while it runs. */
+export interface CrawlProgress {
+    recipes: number;
+    recipesDone: number;
+    records: number;
+    /** Recipes waiting for a posted-back result. Present only when the host takes them. */
+    recipesWaiting?: number;
+}
+//# sourceMappingURL=crawl-job.model.d.ts.map

package/dist/crawl-run/crawl-job.validator.d.ts

@@ -0,0 +1,11 @@
+import type { CrawlJob } from './crawl-job.model';
+/**
+ * Checks a `/crawl` orchestration's input as it comes back out of the task
+ * hub: an instance started before a deploy resumes against the new code.
+ *
+ * @param raw - The stored input.
+ * @returns The job.
+ * @throws ZodError when it is not one.
+ */
+export declare function parseCrawlJob(raw: unknown): CrawlJob;
+//# sourceMappingURL=crawl-job.validator.d.ts.map

package/dist/crawl-run/crawl-orchestration.use-case.d.ts

@@ -0,0 +1,22 @@
+import type { TypedActivity } from '@mnci/az-durable';
+import type { OrchestrationContext, Task } from 'durable-functions';
+import type { CrawlJob, CrawlResult, RecipeTask, RecipeTaskResult } from './crawl-job.model';
+export interface CrawlOrchestratorOptions {
+    /** How long a recipe waits for the results of parked calls before it fails. Only used when a call parks. */
+    waitMs?: number;
+}
+/**
+ * The `/crawl` orchestration: one activity per input recipe, all at once;
+ * the custom status counts recipes and records as each one finishes.
+ *
+ * A recipe whose hooks answered `pending` and will post their results back stops its activity with
+ * `parked` calls. The orchestration then waits for one external event per call (and a timer, so a
+ * handler that never answers fails the recipe, not the job) and runs the recipe again with the results
+ * so far: calls that have a result return it without being made again.
+ *
+ * @param runRecipe - The activity that runs one recipe.
+ * @param options - How long to wait for posted-back results.
+ * @returns The orchestrator body.
+ */
+export declare function crawlOrchestrator(runRecipe: TypedActivity<RecipeTask, RecipeTaskResult>, options?: CrawlOrchestratorOptions): (context: OrchestrationContext, job: CrawlJob) => Generator<Task, CrawlResult, unknown>;
+//# sourceMappingURL=crawl-orchestration.use-case.d.ts.map

package/dist/crawl-run/crawl-request.validator.d.ts

@@ -0,0 +1,25 @@
+import { z } from 'zod';
+/** A `/crawl` body: the recipes themselves, or the name and version of a stored set. */
+export declare const crawlRequestSchema: z.ZodUnion<readonly [z.ZodObject<{
+    output: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    inputs: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    options: z.ZodOptional<z.ZodObject<{
+        dedupe: z.ZodOptional<z.ZodEnum<{
+            off: "off";
+            recipe: "recipe";
+        }>>;
+    }, z.core.$strict>>;
+}, z.core.$strict>, z.ZodObject<{
+    recipe: z.ZodObject<{
+        name: z.ZodString;
+        version: z.ZodString;
+    }, z.core.$strict>;
+    options: z.ZodOptional<z.ZodObject<{
+        dedupe: z.ZodOptional<z.ZodEnum<{
+            off: "off";
+            recipe: "recipe";
+        }>>;
+    }, z.core.$strict>>;
+}, z.core.$strict>]>;
+export type CrawlRequest = z.infer<typeof crawlRequestSchema>;
+//# sourceMappingURL=crawl-request.validator.d.ts.map

package/dist/crawl-run/index.d.ts

@@ -0,0 +1,8 @@
+export { crawlOrchestrator } from './crawl-orchestration.use-case';
+export { runRecipeTask } from './run-recipe.use-case';
+export { startCrawl } from './start-crawl.handler';
+export { crawlRequestSchema } from './crawl-request.validator';
+export type { CrawlRequest } from './crawl-request.validator';
+export type { CrawlJob, CrawlResult, CrawlProgress, RecipeTask, RecipeTaskResult } from './crawl-job.model';
+export { parseCrawlJob } from './crawl-job.validator';
+//# sourceMappingURL=index.d.ts.map

package/dist/crawl-run/run-recipe.use-case.d.ts

@@ -0,0 +1,18 @@
+import type { HostSettings } from '../host-options';
+import type { RecipeTask, RecipeTaskResult } from './crawl-job.model';
+/**
+ * The `/crawl` activity: one input recipe, on a crawler of its own, limited
+ * to the caller's hosts. A recipe that fails is reported, not thrown: the
+ * orchestration still gathers the others.
+ *
+ * When the host takes posted-back results, a hook that answers `pending` parks its call instead of
+ * being polled: the run stops and the result says which calls it is waiting for (`parked`), with no
+ * reports, because the orchestration runs the recipe again once they have arrived.
+ *
+ * @param settings - The host settings.
+ * @param task - The recipe and where its records go.
+ * @param now - The current time, epoch milliseconds (for the token expiry).
+ * @returns Its reports (one per matrix variant) and records (or their link), or what it waits for.
+ */
+export declare function runRecipeTask(settings: HostSettings, task: RecipeTask, now?: number): Promise<RecipeTaskResult>;
+//# sourceMappingURL=run-recipe.use-case.d.ts.map

package/dist/crawl-run/start-crawl.handler.d.ts

@@ -0,0 +1,18 @@
+import type { HttpRequest, HttpResponse, HttpResponseInit } from '@azure/functions';
+import type { TypedOrchestration } from '@mnci/az-durable';
+import type { DurableClient } from 'durable-functions';
+import type { HostSettings } from '../host-options';
+import type { CrawlJob, CrawlResult } from './crawl-job.model';
+/**
+ * `POST /crawl`: checks the caller and the recipes, then starts the
+ * orchestration. Recipes that do not load are a 400 before anything is
+ * queued.
+ *
+ * @param request - The request.
+ * @param client - The Durable client.
+ * @param settings - The host settings.
+ * @param orchestration - The `/crawl` orchestration.
+ * @returns 202 with the Durable status URLs, or the problem.
+ */
+export declare function startCrawl(request: HttpRequest, client: DurableClient, settings: HostSettings, orchestration: TypedOrchestration<CrawlJob, CrawlResult>): Promise<HttpResponse | HttpResponseInit>;
+//# sourceMappingURL=start-crawl.handler.d.ts.map

package/dist/host-options/host-options.config.d.ts

@@ -0,0 +1,100 @@
+import type { HttpRequest } from '@azure/functions';
+import type { AccessConfig, AccessPlugin, BrowserSessionConfig, CaptchaSolver, CrawlOptions, HookMap, WindowsPolicy } from '@opencraw/core';
+import type { RecipeStore } from '../recipe-store';
+import type { ResultStore } from '../result-store';
+/** How the host is set up: everything that is code or a secret lives here, never in a request. */
+export interface OpenCrawHostOptions {
+    /**
+     * The hosts a caller's recipes may reach (`example.com`, `*.example.com`,
+     * `host:port`, `*`). A list applies to every caller; a function decides per
+     * caller, and a caller it returns nothing for is refused (403).
+     */
+    allowedHosts: readonly string[] | ((caller: string | undefined) => readonly string[] | undefined);
+    /**
+     * Who is calling, from the request: an Entra ID principal (App Service
+     * authentication sets `x-ms-client-principal-name`), a key name, a header
+     * your gateway sets. Only trust a header something in front of the app
+     * sets and callers cannot. Default: nobody in particular.
+     */
+    identify?: (request: HttpRequest) => string | undefined;
+    /** Named recipes: what `/jobs` items and `/crawl` by name run. */
+    recipes?: RecipeStore;
+    /** Where results too large to return inline go. Without one, every result is returned inline. */
+    results?: ResultStore;
+    /** Results larger than this (as JSON) go to `results`. Default 256 KiB. */
+    inlineLimitBytes?: number;
+    hooks?: HookMap;
+    /** A fresh set per crawler: a crawler closes its solvers when it closes. */
+    captchaSolvers?: () => CaptchaSolver[];
+    access?: AccessConfig;
+    accessPlugins?: AccessPlugin[];
+    browser?: BrowserSessionConfig;
+    /** Worker pools: one per caller and crawl id. */
+    pools?: PoolSettings;
+    /**
+     * The recipe-authoring MCP endpoint, `/mcp`: `true`, or its sample limits.
+     * Off by default. It refuses to start without authentication.
+     */
+    mcp?: boolean | McpSettings;
+    /** Who may promote a draft to production (`POST /recipes/{name}/{version}/promote`). Default: nobody. */
+    canPromote?: (caller: string | undefined) => boolean;
+    /** Default `function`: a function key is needed. `anonymous` only behind something that authenticates. */
+    authLevel?: 'anonymous' | 'function' | 'admin';
+    /** Prepended to every route: `'opencraw'` gives `/api/opencraw/crawl`. Default none. */
+    routePrefix?: string;
+    /**
+     * Lets a hook that answers `pending` post its result back (`POST /callouts/{token}/resolve`) instead of
+     * being asked again. Off by default: a pending hook is then polled inside the activity.
+     */
+    callouts?: CalloutSettings;
+}
+export interface CalloutSettings {
+    /** The name of the environment variable holding the key that signs resume tokens. A secret: never in code or a recipe. */
+    signingKeyEnv: string;
+    /** The base URL handlers post to (`https://app.example.com/api`), when the request URL is not it (a proxy, a custom domain). Default: derived from the request that started the job. */
+    publicUrl?: string;
+    /** How long a job waits for a posted-back result before the recipe fails. Default 1 hour. */
+    waitMs?: number;
+}
+export interface McpSettings {
+    /** A sample run (`run` without `full`) stops after this many records per input recipe. Default 20. */
+    sampleRecords?: number;
+    /** And after this long, whatever it has by then. Default 60 seconds: MCP clients time out tool calls. */
+    sampleMs?: number;
+}
+export interface PoolSettings {
+    /** A pool with no item for this long closes. Default 10 minutes. */
+    idleTtlMs?: number;
+    /** The windows policy a job gets when its request names none. Default `{ min: 1, max: 4, grow: { after: 5 }, idle: { afterMs: 60000 } }`. */
+    windows?: WindowsPolicy;
+    /** No pool gets more windows than this, whatever a job asks. Default 8. */
+    maxWindows?: number;
+}
+/** The options with their defaults. */
+export interface HostSettings extends Omit<OpenCrawHostOptions, 'pools' | 'inlineLimitBytes' | 'routePrefix' | 'authLevel' | 'mcp' | 'canPromote' | 'callouts'> {
+    inlineLimitBytes: number;
+    routePrefix: string;
+    authLevel: 'anonymous' | 'function' | 'admin';
+    pools: Required<PoolSettings>;
+    /** `undefined` when the endpoint is off. */
+    mcp?: Required<McpSettings>;
+    canPromote: (caller: string | undefined) => boolean;
+    /** `undefined` when posted-back results are off. */
+    callouts?: Required<Pick<CalloutSettings, 'signingKeyEnv' | 'waitMs'>> & Pick<CalloutSettings, 'publicUrl'>;
+}
+/**
+ * @param options - As given.
+ * @returns The options with their defaults.
+ * @throws Error when `allowedHosts` is an empty list (a host that may reach nothing runs nothing), or the MCP endpoint would be open to anyone.
+ */
+export declare function resolveHostOptions(options: OpenCrawHostOptions): HostSettings;
+/**
+ * What a crawler for one caller is created with: the host's code and
+ * secrets, and the caller's hosts.
+ *
+ * @param settings - The host settings.
+ * @param allowedHosts - The caller's hosts.
+ * @returns Crawl options, without a sink.
+ */
+export declare function crawlOptionsFor(settings: HostSettings, allowedHosts: readonly string[]): CrawlOptions;
+//# sourceMappingURL=host-options.config.d.ts.map

package/dist/host-options/index.d.ts

@@ -0,0 +1,3 @@
+export { resolveHostOptions, crawlOptionsFor } from './host-options.config';
+export type { OpenCrawHostOptions, PoolSettings, McpSettings, CalloutSettings, HostSettings } from './host-options.config';
+//# sourceMappingURL=index.d.ts.map

package/dist/http-reply/http-reply.mapper.d.ts

@@ -0,0 +1,24 @@
+import type { HttpRequest, HttpResponseInit } from '@azure/functions';
+/**
+ * @param status - The HTTP status.
+ * @param body - A JSON body.
+ * @returns The response.
+ */
+export declare function reply(status: number, body: unknown): HttpResponseInit;
+/**
+ * An error the caller can act on: what went wrong, and the issues a recipe
+ * error lists (each with its JSON path).
+ *
+ * @param status - The HTTP status.
+ * @param error - What went wrong.
+ * @returns The response.
+ */
+export declare function problem(status: number, error: unknown): HttpResponseInit;
+/**
+ * The request body as JSON.
+ *
+ * @param request - The request.
+ * @returns The body, or `undefined` when it is not JSON.
+ */
+export declare function readJson(request: HttpRequest): Promise<unknown>;
+//# sourceMappingURL=http-reply.mapper.d.ts.map

package/dist/http-reply/index.d.ts

@@ -0,0 +1,2 @@
+export { reply, problem, readJson } from './http-reply.mapper';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts

@@ -1 +1,10 @@
-export * from "./src/index.js";
+export { registerOpenCraw, hostJsonWarnings } from './open-craw-host';
+export type { OpenCrawHost } from './open-craw-host';
+export type { OpenCrawHostOptions, PoolSettings, McpSettings, CalloutSettings, HostSettings } from './host-options';
+export { memoryRecipes, blobRecipes, RecipeStoreError } from './recipe-store';
+export type { RecipeStore, StoredRecipes, BlobRecipesOptions } from './recipe-store';
+export { blobResults } from './result-store';
+export type { ResultStore, BlobResultsOptions } from './result-store';
+export type { CrawlJob, CrawlResult, CrawlProgress, CrawlRequest, RecipeTask, RecipeTaskResult } from './crawl-run';
+export type { ItemJob, ItemOutcome, PoolState, ItemRequest } from './worker-pools';
+//# sourceMappingURL=index.d.ts.map