@opencraw/azure-durable 0.0.48 → 0.0.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/caller-access/caller-access.policy.d.ts +28 -0
  2. package/dist/caller-access/index.d.ts +3 -0
  3. package/dist/callout-resume/callout-event.mapper.d.ts +30 -0
  4. package/dist/callout-resume/callout-ticket.algorithm.d.ts +36 -0
  5. package/dist/callout-resume/index.d.ts +5 -0
  6. package/dist/callout-resume/resolve-callout.handler.d.ts +17 -0
  7. package/dist/crawl-run/crawl-job.model.d.ts +52 -0
  8. package/dist/crawl-run/crawl-job.validator.d.ts +11 -0
  9. package/dist/crawl-run/crawl-orchestration.use-case.d.ts +22 -0
  10. package/dist/crawl-run/crawl-request.validator.d.ts +25 -0
  11. package/dist/crawl-run/index.d.ts +8 -0
  12. package/dist/crawl-run/run-recipe.use-case.d.ts +18 -0
  13. package/dist/crawl-run/start-crawl.handler.d.ts +18 -0
  14. package/dist/host-options/host-options.config.d.ts +100 -0
  15. package/dist/host-options/index.d.ts +3 -0
  16. package/dist/http-reply/http-reply.mapper.d.ts +24 -0
  17. package/dist/http-reply/index.d.ts +2 -0
  18. package/dist/index.d.ts +10 -1
  19. package/dist/index.esm.js +428 -1157
  20. package/dist/open-craw-host/host-json.validator.d.ts +11 -0
  21. package/dist/open-craw-host/index.d.ts +4 -0
  22. package/dist/open-craw-host/register-open-craw.use-case.d.ts +48 -0
  23. package/dist/recipe-authoring/authoring-server.use-case.d.ts +13 -0
  24. package/dist/recipe-authoring/authoring-tools.use-case.d.ts +96 -0
  25. package/dist/recipe-authoring/index.d.ts +5 -0
  26. package/dist/recipe-authoring/mcp-endpoint.handler.d.ts +18 -0
  27. package/dist/recipe-authoring/promote.handler.d.ts +13 -0
  28. package/dist/recipe-store/blob-recipe.store.d.ts +20 -0
  29. package/dist/recipe-store/find-recipes.use-case.d.ts +13 -0
  30. package/dist/recipe-store/index.d.ts +7 -0
  31. package/dist/recipe-store/memory-recipe.store.d.ts +12 -0
  32. package/dist/recipe-store/recipe-store.contract.d.ts +28 -0
  33. package/dist/recipe-store/recipe-store.error.d.ts +6 -0
  34. package/dist/result-store/blob-result.store.d.ts +18 -0
  35. package/dist/result-store/index.d.ts +6 -0
  36. package/dist/result-store/result-packing.policy.d.ts +22 -0
  37. package/dist/result-store/result-store.contract.d.ts +13 -0
  38. package/dist/src/crawl-run/crawl-request.validator.d.ts +2 -2
  39. package/dist/src/worker-pools/item-request.validator.d.ts +1 -1
  40. package/dist/worker-pools/index.d.ts +10 -0
  41. package/dist/worker-pools/item-job.validator.d.ts +10 -0
  42. package/dist/worker-pools/item-orchestration.use-case.d.ts +14 -0
  43. package/dist/worker-pools/item-request.validator.d.ts +35 -0
  44. package/dist/worker-pools/job.handler.d.ts +14 -0
  45. package/dist/worker-pools/pool-job.model.d.ts +43 -0
  46. package/dist/worker-pools/submit-item.handler.d.ts +20 -0
  47. package/dist/worker-pools/windows-cap.policy.d.ts +12 -0
  48. package/dist/worker-pools/worker-pools.store.d.ts +58 -0
  49. package/package.json +10 -38
@@ -0,0 +1,11 @@
1
+ /**
2
+ * What in the Function App's `host.json` would make the host misbehave. An
3
+ * item waiting in a pool's inbox holds an activity slot, so a pool can only
4
+ * fill its windows if Durable runs that many activities at once.
5
+ *
6
+ * @param maxWindows - The most windows one pool may have.
7
+ * @param directory - Where `host.json` is; the working directory by default.
8
+ * @returns Warnings, none when `host.json` is missing or fine.
9
+ */
10
+ export declare function hostJsonWarnings(maxWindows: number, directory?: string): string[];
11
+ //# sourceMappingURL=host-json.validator.d.ts.map
@@ -0,0 +1,4 @@
1
+ export { registerOpenCraw } from './register-open-craw.use-case';
2
+ export type { OpenCrawHost } from './register-open-craw.use-case';
3
+ export { hostJsonWarnings } from './host-json.validator';
4
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,48 @@
1
+ import type { HttpRequest, HttpResponse, HttpResponseInit } from '@azure/functions';
2
+ import type { TypedOrchestration } from '@mnci/az-durable';
3
+ import type { DurableClient } from 'durable-functions';
4
+ import type { CrawlJob, CrawlResult, RecipeTask, RecipeTaskResult } from '../crawl-run';
5
+ import type { HostSettings, OpenCrawHostOptions } from '../host-options';
6
+ import { WorkerPools } from '../worker-pools';
7
+ import type { ItemJob, ItemOutcome } from '../worker-pools';
8
+ /** The functions the host registered, for tests and for an app that wants to call them itself. */
9
+ export interface OpenCrawHost {
10
+ settings: HostSettings;
11
+ pools: WorkerPools;
12
+ orchestrations: {
13
+ crawl: TypedOrchestration<CrawlJob, CrawlResult>;
14
+ item: TypedOrchestration<ItemJob, ItemOutcome>;
15
+ };
16
+ activities: {
17
+ runRecipe: (task: RecipeTask) => Promise<RecipeTaskResult>;
18
+ runItem: (job: ItemJob) => Promise<ItemOutcome>;
19
+ };
20
+ http: {
21
+ startCrawl: (request: HttpRequest, client: DurableClient) => Promise<HttpResponse | HttpResponseInit>;
22
+ /** `POST /callouts/{token}/resolve`; registered only when `callouts` is set. */
23
+ resolveCallout: (request: HttpRequest, client: DurableClient) => Promise<HttpResponseInit>;
24
+ submitItem: (request: HttpRequest, client: DurableClient) => Promise<HttpResponse | HttpResponseInit>;
25
+ job: (request: HttpRequest) => Promise<HttpResponseInit>;
26
+ /** The authoring MCP endpoint; registered only when `mcp` is on. */
27
+ mcp: (request: HttpRequest, client: DurableClient) => Promise<HttpResponseInit>;
28
+ promote: (request: HttpRequest) => Promise<HttpResponseInit>;
29
+ };
30
+ /** Closes every pool; also run when the Function App shuts down. */
31
+ close: () => Promise<void>;
32
+ }
33
+ /**
34
+ * Registers OpenCraw's HTTP API in this Function App (the v4 programming
35
+ * model): `POST /crawl` runs recipes once, `POST /jobs/{crawlId}/items`
36
+ * sends one item to the caller's warm pool, `GET` and `DELETE
37
+ * /jobs/{crawlId}` read and close it, `POST /recipes/{name}/{version}/promote`
38
+ * makes a draft runnable, with `callouts` set `POST /callouts/{token}/resolve`
39
+ * takes the results of hooks that answered `pending`, and with `mcp` on, `/mcp` serves the authoring
40
+ * tools. Call it once, from the app's entry module; nothing is registered by
41
+ * importing the package.
42
+ *
43
+ * @param options - Allowed hosts, callers, recipes, result storage, plugins, pools.
44
+ * @returns The registered pieces.
45
+ * @throws Error when the options cannot work, or when it is called twice (function names are global).
46
+ */
47
+ export declare function registerOpenCraw(options: OpenCrawHostOptions): OpenCrawHost;
48
+ //# sourceMappingURL=register-open-craw.use-case.d.ts.map
@@ -0,0 +1,13 @@
1
+ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
2
+ import type { AuthoringContext } from './authoring-tools.use-case';
3
+ /**
4
+ * The authoring MCP server for one call: the tools an agent writes recipes
5
+ * with, run where the recipes will run, for this caller only. Unlike the
6
+ * local `@opencraw/mcp`, nothing takes a path: recipes travel inline, and a
7
+ * finished set is published to the host's store.
8
+ *
9
+ * @param context - The host, the caller, Durable.
10
+ * @returns The server, ready to `connect`.
11
+ */
12
+ export declare function createAuthoringServer(context: AuthoringContext): McpServer;
13
+ //# sourceMappingURL=authoring-server.use-case.d.ts.map
@@ -0,0 +1,96 @@
1
+ import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
2
+ import type { TypedOrchestration } from '@mnci/az-durable';
3
+ import type { DurableClient } from 'durable-functions';
4
+ import type { CallerAccess } from '../caller-access';
5
+ import type { CrawlJob, CrawlResult } from '../crawl-run';
6
+ import type { HostSettings } from '../host-options';
7
+ /** What every authoring tool call works with: the host, the caller, and Durable for full runs. */
8
+ export interface AuthoringContext {
9
+ settings: HostSettings & {
10
+ mcp: NonNullable<HostSettings['mcp']>;
11
+ };
12
+ access: CallerAccess;
13
+ client: DurableClient;
14
+ crawl: TypedOrchestration<CrawlJob, CrawlResult>;
15
+ }
16
+ /**
17
+ * `probe`: where a page's data lives, fetched from the host itself (its IP,
18
+ * its proxies) and only from the caller's hosts.
19
+ *
20
+ * @param context - The call's context.
21
+ * @param args - The page, whether to render it, an access profile.
22
+ * @returns The findings.
23
+ */
24
+ export declare function probe(context: AuthoringContext, args: {
25
+ url: string;
26
+ browser?: boolean;
27
+ access?: string;
28
+ }): Promise<CallToolResult>;
29
+ /**
30
+ * `validate`: `@opencraw/mcp`'s own, on inline recipes only (never paths on the host).
31
+ *
32
+ * @param args - The recipes.
33
+ * @returns Every issue with its JSON path.
34
+ */
35
+ export declare function validate(args: {
36
+ recipes: string | Record<string, unknown>[];
37
+ }): Promise<CallToolResult>;
38
+ /**
39
+ * `run`: a sample by default (a few records per input recipe, within the
40
+ * time a tool call may take), or with `full` a Durable crawl whose instance
41
+ * id `status` reads. Drafts run here; production routes refuse them.
42
+ *
43
+ * @param context - The call's context.
44
+ * @param args - Inline recipes or a stored set, and whether to run it all.
45
+ * @returns The sample's reports and records, or the instance id.
46
+ */
47
+ export declare function run(context: AuthoringContext, args: {
48
+ recipes?: Record<string, unknown>[];
49
+ recipe?: {
50
+ name: string;
51
+ version: string;
52
+ };
53
+ full?: boolean;
54
+ }): Promise<CallToolResult>;
55
+ /**
56
+ * `status`: a full run's state and result, for the caller who started it.
57
+ *
58
+ * @param context - The call's context.
59
+ * @param args - The instance id `run` returned.
60
+ * @returns Its runtime status, progress and output.
61
+ */
62
+ export declare function status(context: AuthoringContext, args: {
63
+ instanceId: string;
64
+ }): Promise<CallToolResult>;
65
+ /**
66
+ * `list_recipes`: the stored recipe sets, with their versions and states.
67
+ *
68
+ * @param context - The call's context.
69
+ * @returns The list.
70
+ */
71
+ export declare function listRecipes(context: AuthoringContext): Promise<CallToolResult>;
72
+ /**
73
+ * `get_recipes`: a stored set's recipes, drafts included: the start of a new version.
74
+ *
75
+ * @param context - The call's context.
76
+ * @param args - The name and version.
77
+ * @returns The set.
78
+ */
79
+ export declare function getRecipes(context: AuthoringContext, args: {
80
+ name: string;
81
+ version: string;
82
+ }): Promise<CallToolResult>;
83
+ /**
84
+ * `publish`: saves recipes that load as a new version, as a draft. A person
85
+ * (or a caller allowed to) promotes it before production runs it.
86
+ *
87
+ * @param context - The call's context.
88
+ * @param args - The name, a new version, the recipes.
89
+ * @returns The stored version.
90
+ */
91
+ export declare function publish(context: AuthoringContext, args: {
92
+ name: string;
93
+ version: string;
94
+ recipes: Record<string, unknown>[];
95
+ }): Promise<CallToolResult>;
96
+ //# sourceMappingURL=authoring-tools.use-case.d.ts.map
@@ -0,0 +1,5 @@
1
+ export { mcpEndpoint } from './mcp-endpoint.handler';
2
+ export { promoteRecipes } from './promote.handler';
3
+ export { createAuthoringServer } from './authoring-server.use-case';
4
+ export type { AuthoringContext } from './authoring-tools.use-case';
5
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,18 @@
1
+ import type { HttpRequest, HttpResponseInit } from '@azure/functions';
2
+ import type { TypedOrchestration } from '@mnci/az-durable';
3
+ import type { DurableClient } from 'durable-functions';
4
+ import type { CrawlJob, CrawlResult } from '../crawl-run';
5
+ import type { HostSettings } from '../host-options';
6
+ /**
7
+ * `/mcp`: the authoring MCP server over Streamable HTTP, stateless: every
8
+ * request gets its own server and transport, and a JSON answer, so any
9
+ * instance can take any call.
10
+ *
11
+ * @param request - The request.
12
+ * @param client - The Durable client, for full runs.
13
+ * @param settings - The host settings.
14
+ * @param crawl - The `/crawl` orchestration.
15
+ * @returns The MCP answer, 403 for a caller with no hosts, 404 when the endpoint is off.
16
+ */
17
+ export declare function mcpEndpoint(request: HttpRequest, client: DurableClient, settings: HostSettings, crawl: TypedOrchestration<CrawlJob, CrawlResult>): Promise<HttpResponseInit>;
18
+ //# sourceMappingURL=mcp-endpoint.handler.d.ts.map
@@ -0,0 +1,13 @@
1
+ import type { HttpRequest, HttpResponseInit } from '@azure/functions';
2
+ import type { HostSettings } from '../host-options';
3
+ /**
4
+ * `POST /recipes/{name}/{version}/promote`: a draft becomes runnable in
5
+ * production. Only for callers `canPromote` allows: an agent publishes, a
6
+ * person promotes.
7
+ *
8
+ * @param request - The request.
9
+ * @param settings - The host settings.
10
+ * @returns The promoted version, 403, or the store's 404.
11
+ */
12
+ export declare function promoteRecipes(request: HttpRequest, settings: HostSettings): Promise<HttpResponseInit>;
13
+ //# sourceMappingURL=promote.handler.d.ts.map
@@ -0,0 +1,20 @@
1
+ import type { RecipeStore, StoredRecipes } from './recipe-store.contract';
2
+ export interface BlobRecipesOptions {
3
+ /** A storage connection string. */
4
+ connectionString: string;
5
+ /** Default `opencraw-recipes`; created when missing. */
6
+ container?: string;
7
+ /** The recipe sets the deployment ships with: read-only, promoted, and never shadowed by a publish. */
8
+ shipped?: Omit<StoredRecipes, 'state'>[];
9
+ }
10
+ /**
11
+ * Named recipes in Azure Blob Storage: one JSON blob per version, its state
12
+ * (`draft` or `promoted`) in the blob's metadata. A version is written once:
13
+ * a second publish of it fails even when two race, since the write only
14
+ * succeeds when no blob is there.
15
+ *
16
+ * @param options - The account, the container, and the sets the deployment ships with.
17
+ * @returns The store.
18
+ */
19
+ export declare function blobRecipes(options: BlobRecipesOptions): RecipeStore;
20
+ //# sourceMappingURL=blob-recipe.store.d.ts.map
@@ -0,0 +1,13 @@
1
+ import type { RecipeStore, StoredRecipes } from './recipe-store.contract';
2
+ /**
3
+ * A stored recipe set production may run.
4
+ *
5
+ * @param store - The store, if the host has one.
6
+ * @param name - The set's name.
7
+ * @param version - Its version.
8
+ * @param allowDraft - Whether a draft will do (authoring tools), or only a promoted version (production).
9
+ * @returns The set.
10
+ * @throws RecipeStoreError 404 when there is no such version, 409 when it is a draft and a draft won't do.
11
+ */
12
+ export declare function findRecipes(store: RecipeStore | undefined, name: string, version: string, allowDraft?: boolean): Promise<StoredRecipes>;
13
+ //# sourceMappingURL=find-recipes.use-case.d.ts.map
@@ -0,0 +1,7 @@
1
+ export { RecipeStoreError } from './recipe-store.error';
2
+ export type { RecipeStore, StoredRecipes } from './recipe-store.contract';
3
+ export { memoryRecipes } from './memory-recipe.store';
4
+ export { findRecipes } from './find-recipes.use-case';
5
+ export { blobRecipes } from './blob-recipe.store';
6
+ export type { BlobRecipesOptions } from './blob-recipe.store';
7
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,12 @@
1
+ import type { RecipeStore, StoredRecipes } from './recipe-store.contract';
2
+ /**
3
+ * Recipes held in memory: the ones a deployment ships with (promoted), plus
4
+ * drafts published while it runs, which a restart forgets.
5
+ *
6
+ * @param entries - The recipe sets; `state` defaults to `promoted`.
7
+ * @returns The store.
8
+ */
9
+ export declare function memoryRecipes(entries?: (Omit<StoredRecipes, 'state'> & {
10
+ state?: StoredRecipes['state'];
11
+ })[]): RecipeStore;
12
+ //# sourceMappingURL=memory-recipe.store.d.ts.map
@@ -0,0 +1,28 @@
1
+ /** A named, versioned recipe set: its output recipe and its input recipes. */
2
+ export interface StoredRecipes {
3
+ name: string;
4
+ /** Any string: `3`, `2026-09-27`. A version, once written, never changes. */
5
+ version: string;
6
+ /** The recipes as JSON objects: one output recipe and its input recipes, in any order. */
7
+ recipes: unknown[];
8
+ /** A draft can be tried, but production routes (`/crawl` by name, `/jobs`) refuse it. */
9
+ state: 'draft' | 'promoted';
10
+ }
11
+ /** Where the host finds named recipes. */
12
+ export interface RecipeStore {
13
+ get(name: string, version: string): Promise<StoredRecipes | undefined>;
14
+ list(): Promise<Omit<StoredRecipes, 'recipes'>[]>;
15
+ /**
16
+ * Saves a new version as a draft.
17
+ *
18
+ * @throws Error when that version exists: versions never change.
19
+ */
20
+ put(entry: Omit<StoredRecipes, 'state'>): Promise<void>;
21
+ /**
22
+ * Makes a draft runnable in production.
23
+ *
24
+ * @throws Error when there is no such version.
25
+ */
26
+ promote(name: string, version: string): Promise<void>;
27
+ }
28
+ //# sourceMappingURL=recipe-store.contract.d.ts.map
@@ -0,0 +1,6 @@
1
+ /** A version that does not exist, or a write to one that does. */
2
+ export declare class RecipeStoreError extends Error {
3
+ readonly status: 404 | 409;
4
+ constructor(message: string, status: 404 | 409);
5
+ }
6
+ //# sourceMappingURL=recipe-store.error.d.ts.map
@@ -0,0 +1,18 @@
1
+ import type { ResultStore } from './result-store.contract';
2
+ export interface BlobResultsOptions {
3
+ /** A storage connection string with an account key (the key signs the read links). */
4
+ connectionString: string;
5
+ /** Default `opencraw-results`; created when missing. */
6
+ container?: string;
7
+ /** How long a read link works. Default 24 hours. */
8
+ linkTtlMs?: number;
9
+ }
10
+ /**
11
+ * Results in Azure Blob Storage, as JSON Lines, returned as read-only SAS
12
+ * links that expire.
13
+ *
14
+ * @param options - The account, container and link lifetime.
15
+ * @returns The store.
16
+ */
17
+ export declare function blobResults(options: BlobResultsOptions): ResultStore;
18
+ //# sourceMappingURL=blob-result.store.d.ts.map
@@ -0,0 +1,6 @@
1
+ export type { ResultStore } from './result-store.contract';
2
+ export { blobResults } from './blob-result.store';
3
+ export type { BlobResultsOptions } from './blob-result.store';
4
+ export { packRecords } from './result-packing.policy';
5
+ export type { PackedRecords } from './result-packing.policy';
6
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,22 @@
1
+ import type { OutputRecord } from '@opencraw/core';
2
+ import type { ResultStore } from './result-store.contract';
3
+ /** Records returned in the Durable output, or where to read them when they were too large. */
4
+ export interface PackedRecords {
5
+ records?: OutputRecord[];
6
+ resultUrl?: string;
7
+ }
8
+ /**
9
+ * Returns records inline when small, else saves them and returns a link: an
10
+ * orchestration's history holds every activity output, so a large result
11
+ * belongs in storage, not in the task hub.
12
+ *
13
+ * @param records - The records.
14
+ * @param options - The store (without one, records are always inline), the limit, the name to save under.
15
+ * @returns The records or their link.
16
+ */
17
+ export declare function packRecords(records: readonly OutputRecord[], options: {
18
+ store?: ResultStore;
19
+ inlineLimitBytes: number;
20
+ name: string;
21
+ }): Promise<PackedRecords>;
22
+ //# sourceMappingURL=result-packing.policy.d.ts.map
@@ -0,0 +1,13 @@
1
+ import type { OutputRecord } from '@opencraw/core';
2
+ /** Where results too large to return inline go. */
3
+ export interface ResultStore {
4
+ /**
5
+ * Saves records as JSON Lines.
6
+ *
7
+ * @param name - A path-like name, unique per result: `<instance>/<recipe>.jsonl`.
8
+ * @param records - The records.
9
+ * @returns A URL the caller can read them from.
10
+ */
11
+ save(name: string, records: readonly OutputRecord[]): Promise<string>;
12
+ }
13
+ //# sourceMappingURL=result-store.contract.d.ts.map
@@ -5,8 +5,8 @@ export declare const crawlRequestSchema: z.ZodUnion<readonly [z.ZodObject<{
5
5
  inputs: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
6
6
  options: z.ZodOptional<z.ZodObject<{
7
7
  dedupe: z.ZodOptional<z.ZodEnum<{
8
- recipe: "recipe";
9
8
  off: "off";
9
+ recipe: "recipe";
10
10
  }>>;
11
11
  }, z.core.$strict>>;
12
12
  }, z.core.$strict>, z.ZodObject<{
@@ -16,8 +16,8 @@ export declare const crawlRequestSchema: z.ZodUnion<readonly [z.ZodObject<{
16
16
  }, z.core.$strict>;
17
17
  options: z.ZodOptional<z.ZodObject<{
18
18
  dedupe: z.ZodOptional<z.ZodEnum<{
19
- recipe: "recipe";
20
19
  off: "off";
20
+ recipe: "recipe";
21
21
  }>>;
22
22
  }, z.core.$strict>>;
23
23
  }, z.core.$strict>]>;
@@ -20,8 +20,8 @@ export declare const itemRequestSchema: z.ZodObject<{
20
20
  after: z.ZodOptional<z.ZodNumber>;
21
21
  }, z.core.$strict>>;
22
22
  shrink: z.ZodOptional<z.ZodEnum<{
23
- one: "one";
24
23
  half: "half";
24
+ one: "one";
25
25
  }>>;
26
26
  restart: z.ZodOptional<z.ZodObject<{
27
27
  after: z.ZodOptional<z.ZodNumber>;
@@ -0,0 +1,10 @@
1
+ export { WorkerPools } from './worker-pools.store';
2
+ export { itemOrchestrator } from './item-orchestration.use-case';
3
+ export { submitItem } from './submit-item.handler';
4
+ export { jobRequest } from './job.handler';
5
+ export { capWindows } from './windows-cap.policy';
6
+ export { itemRequestSchema, crawlIdSchema } from './item-request.validator';
7
+ export type { ItemRequest } from './item-request.validator';
8
+ export type { ItemJob, ItemOutcome, PoolState } from './pool-job.model';
9
+ export { parseItemJob } from './item-job.validator';
10
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,10 @@
1
+ import type { ItemJob } from './pool-job.model';
2
+ /**
3
+ * Checks a pooled item's orchestration input as it comes back out of the task hub.
4
+ *
5
+ * @param raw - The stored input.
6
+ * @returns The job.
7
+ * @throws ZodError when it is not one.
8
+ */
9
+ export declare function parseItemJob(raw: unknown): ItemJob;
10
+ //# sourceMappingURL=item-job.validator.d.ts.map
@@ -0,0 +1,14 @@
1
+ import type { TypedActivity } from '@mnci/az-durable';
2
+ import type { OrchestrationContext, Task } from 'durable-functions';
3
+ import type { ItemJob, ItemOutcome } from './pool-job.model';
4
+ /**
5
+ * A pooled item's orchestration: one activity, which hands the item to the
6
+ * warm pool and waits for its result. Durable adds what a plain HTTP call
7
+ * lacks: a status URL to poll past the HTTP time limit, and an instance id
8
+ * that refuses a duplicate while the item runs.
9
+ *
10
+ * @param runItem - The activity.
11
+ * @returns The orchestrator body.
12
+ */
13
+ export declare function itemOrchestrator(runItem: TypedActivity<ItemJob, ItemOutcome>): (context: OrchestrationContext, job: ItemJob) => Generator<Task, ItemOutcome, unknown>;
14
+ //# sourceMappingURL=item-orchestration.use-case.d.ts.map
@@ -0,0 +1,35 @@
1
+ import { z } from 'zod';
2
+ export declare const crawlIdSchema: z.ZodString;
3
+ /** A `/jobs/{crawlId}/items` body. */
4
+ export declare const itemRequestSchema: z.ZodObject<{
5
+ recipe: z.ZodObject<{
6
+ name: z.ZodString;
7
+ version: z.ZodString;
8
+ }, z.core.$strict>;
9
+ item: z.ZodObject<{
10
+ id: z.ZodString;
11
+ vars: z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
12
+ recipe: z.ZodOptional<z.ZodString>;
13
+ }, z.core.$strict>;
14
+ jobSize: z.ZodOptional<z.ZodNumber>;
15
+ windows: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodObject<{
16
+ min: z.ZodOptional<z.ZodNumber>;
17
+ max: z.ZodOptional<z.ZodNumber>;
18
+ start: z.ZodOptional<z.ZodNumber>;
19
+ grow: z.ZodOptional<z.ZodObject<{
20
+ after: z.ZodOptional<z.ZodNumber>;
21
+ }, z.core.$strict>>;
22
+ shrink: z.ZodOptional<z.ZodEnum<{
23
+ half: "half";
24
+ one: "one";
25
+ }>>;
26
+ restart: z.ZodOptional<z.ZodObject<{
27
+ after: z.ZodOptional<z.ZodNumber>;
28
+ }, z.core.$strict>>;
29
+ idle: z.ZodOptional<z.ZodObject<{
30
+ afterMs: z.ZodNumber;
31
+ }, z.core.$strict>>;
32
+ }, z.core.$strict>]>>;
33
+ }, z.core.$strict>;
34
+ export type ItemRequest = z.infer<typeof itemRequestSchema>;
35
+ //# sourceMappingURL=item-request.validator.d.ts.map
@@ -0,0 +1,14 @@
1
+ import type { HttpRequest, HttpResponseInit } from '@azure/functions';
2
+ import type { HostSettings } from '../host-options';
3
+ import type { WorkerPools } from './worker-pools.store';
4
+ /**
5
+ * `GET /jobs/{crawlId}`: the pool's state. `DELETE /jobs/{crawlId}`: closes
6
+ * it; queued and running items finish first.
7
+ *
8
+ * @param request - The request.
9
+ * @param settings - The host settings.
10
+ * @param pools - This process's pools.
11
+ * @returns The state, 202 when closing, 404 when there is no pool.
12
+ */
13
+ export declare function jobRequest(request: HttpRequest, settings: HostSettings, pools: WorkerPools): Promise<HttpResponseInit>;
14
+ //# sourceMappingURL=job.handler.d.ts.map
@@ -0,0 +1,43 @@
1
+ import type { OutputRecord, RecipeReport, WindowsPolicy, WorkItem, WorkOutcome } from '@opencraw/core';
2
+ /** What a pooled item's orchestration runs: written by the starter, never by the caller. */
3
+ export interface ItemJob {
4
+ /** The caller and crawl id: one pool per key. */
5
+ key: string;
6
+ crawlId: string;
7
+ allowedHosts: string[];
8
+ recipe: {
9
+ name: string;
10
+ version: string;
11
+ };
12
+ item: WorkItem;
13
+ jobSize?: number;
14
+ windows?: WindowsPolicy | number;
15
+ /** Where the item's records go when too large to return. */
16
+ resultName: string;
17
+ }
18
+ /** A pool as a caller sees it. */
19
+ export interface PoolState {
20
+ crawlId: string;
21
+ recipe: {
22
+ name: string;
23
+ version: string;
24
+ };
25
+ /** The windows the pool aims for. */
26
+ windows: number;
27
+ /** Windows waiting for an item: while there are any, the caller could send more. */
28
+ idle: number;
29
+ queued: number;
30
+ running: number;
31
+ ended: Record<WorkOutcome, number>;
32
+ closing: boolean;
33
+ }
34
+ /** How a pooled item ended. `refused`: the pool would not take it (another recipe version, a missing or draft recipe set). */
35
+ export interface ItemOutcome {
36
+ outcome: WorkOutcome | 'refused';
37
+ report?: RecipeReport;
38
+ records?: OutputRecord[];
39
+ resultUrl?: string;
40
+ error?: string;
41
+ pool?: PoolState;
42
+ }
43
+ //# sourceMappingURL=pool-job.model.d.ts.map
@@ -0,0 +1,20 @@
1
+ import type { HttpRequest, HttpResponse, HttpResponseInit } from '@azure/functions';
2
+ import type { TypedOrchestration } from '@mnci/az-durable';
3
+ import type { DurableClient } from 'durable-functions';
4
+ import type { HostSettings } from '../host-options';
5
+ import type { ItemJob, ItemOutcome } from './pool-job.model';
6
+ import type { WorkerPools } from './worker-pools.store';
7
+ /**
8
+ * `POST /jobs/{crawlId}/items`: one item for the caller's warm pool. The
9
+ * Durable instance id is the caller, crawl id and item id, so the same item
10
+ * sent again while it runs returns the running instance's status.
11
+ *
12
+ * @param request - The request.
13
+ * @param client - The Durable client.
14
+ * @param settings - The host settings.
15
+ * @param pools - This process's pools.
16
+ * @param orchestration - The item orchestration.
17
+ * @returns 202 with the Durable status URLs, or the problem.
18
+ */
19
+ export declare function submitItem(request: HttpRequest, client: DurableClient, settings: HostSettings, pools: WorkerPools, orchestration: TypedOrchestration<ItemJob, ItemOutcome>): Promise<HttpResponse | HttpResponseInit>;
20
+ //# sourceMappingURL=submit-item.handler.d.ts.map
@@ -0,0 +1,12 @@
1
+ import type { WindowsPolicy } from '@opencraw/core';
2
+ /**
3
+ * A job's windows, within the host's limit and never more than the job has
4
+ * items: twelve windows for three items would only sit idle.
5
+ *
6
+ * @param policy - What the job asked for.
7
+ * @param maxWindows - The host's limit.
8
+ * @param jobSize - About how many items the job has.
9
+ * @returns The policy, capped.
10
+ */
11
+ export declare function capWindows(policy: WindowsPolicy | number, maxWindows: number, jobSize?: number): WindowsPolicy;
12
+ //# sourceMappingURL=windows-cap.policy.d.ts.map
@@ -0,0 +1,58 @@
1
+ import type { HostSettings } from '../host-options';
2
+ import type { ItemJob, ItemOutcome, PoolState } from './pool-job.model';
3
+ /**
4
+ * The warm worker pools of this process, one per caller and crawl id. A pool
5
+ * is created by its first item, runs `crawler.work` on an inbox, and closes
6
+ * when its job size is reached, when it has had no item for `idleTtlMs`, or
7
+ * when asked. Pools live in this process's memory: a Function App scaled to
8
+ * several instances has one pool per instance for the same crawl id.
9
+ */
10
+ export declare class WorkerPools {
11
+ private readonly settings;
12
+ private readonly now;
13
+ private readonly pools;
14
+ /** Each pool's recipe version, known from the moment it starts opening. */
15
+ private readonly versions;
16
+ private readonly live;
17
+ private sweeper;
18
+ constructor(settings: HostSettings, now?: () => number);
19
+ private open;
20
+ private create;
21
+ /** Waits for a pool to run dry, then closes its browser and forgets it; a pool that failed rejects what it still held. */
22
+ private work;
23
+ private forget;
24
+ /** Starts looking for pools that have had no item for `idleTtlMs`, while any is open. */
25
+ private sweep;
26
+ /**
27
+ * The recipe version a pool runs, if one is open under this key.
28
+ *
29
+ * @param key - The caller and crawl id.
30
+ * @returns The version.
31
+ */
32
+ versionOf(key: string): {
33
+ name: string;
34
+ version: string;
35
+ } | undefined;
36
+ /**
37
+ * @param key - The caller and crawl id.
38
+ * @returns The pool as a caller sees it, if one is open.
39
+ */
40
+ state(key: string): PoolState | undefined;
41
+ /**
42
+ * Runs an item on its job's pool, opening the pool when it is the first.
43
+ *
44
+ * @param job - The item and its job.
45
+ * @returns How it ended, with the pool's state.
46
+ */
47
+ run(job: ItemJob): Promise<ItemOutcome>;
48
+ /**
49
+ * Closes a pool: queued and running items finish, then its browser closes.
50
+ *
51
+ * @param key - The caller and crawl id.
52
+ * @returns Whether there was a pool to close.
53
+ */
54
+ close(key: string): Promise<boolean>;
55
+ /** Closes every pool, as the host shuts down. */
56
+ closeAll(): Promise<void>;
57
+ }
58
+ //# sourceMappingURL=worker-pools.store.d.ts.map