@opencraw/azure-durable 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +125 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.esm.js +1451 -0
- package/dist/src/caller-access/caller-access.policy.d.ts +28 -0
- package/dist/src/caller-access/index.d.ts +3 -0
- package/dist/src/crawl-run/crawl-job.model.d.ts +40 -0
- package/dist/src/crawl-run/crawl-job.validator.d.ts +11 -0
- package/dist/src/crawl-run/crawl-orchestration.use-case.d.ts +12 -0
- package/dist/src/crawl-run/crawl-request.validator.d.ts +25 -0
- package/dist/src/crawl-run/index.d.ts +8 -0
- package/dist/src/crawl-run/run-recipe.use-case.d.ts +13 -0
- package/dist/src/crawl-run/start-crawl.handler.d.ts +18 -0
- package/dist/src/host-options/host-options.config.d.ts +85 -0
- package/dist/src/host-options/index.d.ts +3 -0
- package/dist/src/http-reply/http-reply.mapper.d.ts +24 -0
- package/dist/src/http-reply/index.d.ts +2 -0
- package/dist/src/index.d.ts +10 -0
- package/dist/src/open-craw-host/host-json.validator.d.ts +11 -0
- package/dist/src/open-craw-host/index.d.ts +4 -0
- package/dist/src/open-craw-host/register-open-craw.use-case.d.ts +45 -0
- package/dist/src/recipe-authoring/authoring-server.use-case.d.ts +13 -0
- package/dist/src/recipe-authoring/authoring-tools.use-case.d.ts +96 -0
- package/dist/src/recipe-authoring/index.d.ts +5 -0
- package/dist/src/recipe-authoring/mcp-endpoint.handler.d.ts +18 -0
- package/dist/src/recipe-authoring/promote.handler.d.ts +13 -0
- package/dist/src/recipe-store/blob-recipe.store.d.ts +20 -0
- package/dist/src/recipe-store/find-recipes.use-case.d.ts +13 -0
- package/dist/src/recipe-store/index.d.ts +7 -0
- package/dist/src/recipe-store/memory-recipe.store.d.ts +12 -0
- package/dist/src/recipe-store/recipe-store.contract.d.ts +28 -0
- package/dist/src/recipe-store/recipe-store.error.d.ts +6 -0
- package/dist/src/result-store/blob-result.store.d.ts +18 -0
- package/dist/src/result-store/index.d.ts +6 -0
- package/dist/src/result-store/result-packing.policy.d.ts +22 -0
- package/dist/src/result-store/result-store.contract.d.ts +13 -0
- package/dist/src/worker-pools/index.d.ts +10 -0
- package/dist/src/worker-pools/item-job.validator.d.ts +10 -0
- package/dist/src/worker-pools/item-orchestration.use-case.d.ts +14 -0
- package/dist/src/worker-pools/item-request.validator.d.ts +35 -0
- package/dist/src/worker-pools/job.handler.d.ts +14 -0
- package/dist/src/worker-pools/pool-job.model.d.ts +43 -0
- package/dist/src/worker-pools/submit-item.handler.d.ts +20 -0
- package/dist/src/worker-pools/windows-cap.policy.d.ts +12 -0
- package/dist/src/worker-pools/worker-pools.store.d.ts +58 -0
- package/package.json +93 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { HttpRequest } from '@azure/functions';
|
|
2
|
+
/** Who is calling and where their recipes may go. */
|
|
3
|
+
export interface CallerAccess {
|
|
4
|
+
/** `undefined` when the host does not tell callers apart. */
|
|
5
|
+
caller?: string;
|
|
6
|
+
allowedHosts: readonly string[];
|
|
7
|
+
}
|
|
8
|
+
export interface CallerRules {
|
|
9
|
+
allowedHosts: readonly string[] | ((caller: string | undefined) => readonly string[] | undefined);
|
|
10
|
+
identify?: (request: HttpRequest) => string | undefined;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Resolves the caller of a request and the hosts its recipes may reach.
|
|
14
|
+
*
|
|
15
|
+
* @param request - The request.
|
|
16
|
+
* @param rules - The host's `identify` and `allowedHosts`.
|
|
17
|
+
* @returns The access, or `undefined` when this caller may run nothing.
|
|
18
|
+
*/
|
|
19
|
+
export declare function callerAccess(request: HttpRequest, rules: CallerRules): CallerAccess | undefined;
|
|
20
|
+
/**
|
|
21
|
+
* The key a caller's pools and jobs live under, so two callers never share a pool.
|
|
22
|
+
*
|
|
23
|
+
* @param access - The caller.
|
|
24
|
+
* @param crawlId - The job.
|
|
25
|
+
* @returns The key.
|
|
26
|
+
*/
|
|
27
|
+
export declare function poolKey(access: Pick<CallerAccess, 'caller'>, crawlId: string): string;
|
|
28
|
+
//# sourceMappingURL=caller-access.policy.d.ts.map
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { OutputRecord, RecipeReport } from '@opencraw/core';
|
|
2
|
+
/** What a `/crawl` orchestration runs: written by the starter, never by the caller. */
|
|
3
|
+
export interface CrawlJob {
|
|
4
|
+
caller?: string;
|
|
5
|
+
allowedHosts: string[];
|
|
6
|
+
output: unknown;
|
|
7
|
+
inputs: unknown[];
|
|
8
|
+
dedupe?: 'recipe' | 'off';
|
|
9
|
+
}
|
|
10
|
+
/** One input recipe of a job, run by one activity. */
|
|
11
|
+
export interface RecipeTask {
|
|
12
|
+
allowedHosts: string[];
|
|
13
|
+
output: unknown;
|
|
14
|
+
input: unknown;
|
|
15
|
+
dedupe?: 'recipe' | 'off';
|
|
16
|
+
/** Where its records go when too large to return: `<instance>/<index>.jsonl`. */
|
|
17
|
+
resultName: string;
|
|
18
|
+
}
|
|
19
|
+
export interface RecipeTaskResult {
|
|
20
|
+
/** One per run: a recipe with a `matrix` runs once per variant. */
|
|
21
|
+
reports: RecipeReport[];
|
|
22
|
+
records?: OutputRecord[];
|
|
23
|
+
resultUrl?: string;
|
|
24
|
+
}
|
|
25
|
+
/** What a `/crawl` orchestration returns. */
|
|
26
|
+
export interface CrawlResult {
|
|
27
|
+
/** One report per input recipe run (a matrix recipe: per variant), in the order given. */
|
|
28
|
+
recipes: RecipeReport[];
|
|
29
|
+
/** The records returned inline. */
|
|
30
|
+
records: OutputRecord[];
|
|
31
|
+
/** Links to the records too large to return inline, one per recipe that had them. */
|
|
32
|
+
results: string[];
|
|
33
|
+
}
|
|
34
|
+
/** The orchestration's custom status while it runs. */
|
|
35
|
+
export interface CrawlProgress {
|
|
36
|
+
recipes: number;
|
|
37
|
+
recipesDone: number;
|
|
38
|
+
records: number;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=crawl-job.model.d.ts.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { CrawlJob } from './crawl-job.model.js';
|
|
2
|
+
/**
|
|
3
|
+
* Checks a `/crawl` orchestration's input as it comes back out of the task
|
|
4
|
+
* hub: an instance started before a deploy resumes against the new code.
|
|
5
|
+
*
|
|
6
|
+
* @param raw - The stored input.
|
|
7
|
+
* @returns The job.
|
|
8
|
+
* @throws ZodError when it is not one.
|
|
9
|
+
*/
|
|
10
|
+
export declare function parseCrawlJob(raw: unknown): CrawlJob;
|
|
11
|
+
//# sourceMappingURL=crawl-job.validator.d.ts.map
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { TypedActivity } from '@mnci/az-durable';
|
|
2
|
+
import type { OrchestrationContext, Task } from 'durable-functions';
|
|
3
|
+
import type { CrawlJob, CrawlResult, RecipeTask, RecipeTaskResult } from './crawl-job.model.js';
|
|
4
|
+
/**
|
|
5
|
+
* The `/crawl` orchestration: one activity per input recipe, all at once;
|
|
6
|
+
* the custom status counts recipes and records as each one finishes.
|
|
7
|
+
*
|
|
8
|
+
* @param runRecipe - The activity that runs one recipe.
|
|
9
|
+
* @returns The orchestrator body.
|
|
10
|
+
*/
|
|
11
|
+
export declare function crawlOrchestrator(runRecipe: TypedActivity<RecipeTask, RecipeTaskResult>): (context: OrchestrationContext, job: CrawlJob) => Generator<Task, CrawlResult, unknown>;
|
|
12
|
+
//# sourceMappingURL=crawl-orchestration.use-case.d.ts.map
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/** A `/crawl` body: the recipes themselves, or the name and version of a stored set. */
|
|
3
|
+
export declare const crawlRequestSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
4
|
+
output: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
5
|
+
inputs: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
6
|
+
options: z.ZodOptional<z.ZodObject<{
|
|
7
|
+
dedupe: z.ZodOptional<z.ZodEnum<{
|
|
8
|
+
recipe: "recipe";
|
|
9
|
+
off: "off";
|
|
10
|
+
}>>;
|
|
11
|
+
}, z.core.$strict>>;
|
|
12
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
13
|
+
recipe: z.ZodObject<{
|
|
14
|
+
name: z.ZodString;
|
|
15
|
+
version: z.ZodString;
|
|
16
|
+
}, z.core.$strict>;
|
|
17
|
+
options: z.ZodOptional<z.ZodObject<{
|
|
18
|
+
dedupe: z.ZodOptional<z.ZodEnum<{
|
|
19
|
+
recipe: "recipe";
|
|
20
|
+
off: "off";
|
|
21
|
+
}>>;
|
|
22
|
+
}, z.core.$strict>>;
|
|
23
|
+
}, z.core.$strict>]>;
|
|
24
|
+
export type CrawlRequest = z.infer<typeof crawlRequestSchema>;
|
|
25
|
+
//# sourceMappingURL=crawl-request.validator.d.ts.map
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { crawlOrchestrator } from './crawl-orchestration.use-case.js';
|
|
2
|
+
export { runRecipeTask } from './run-recipe.use-case.js';
|
|
3
|
+
export { startCrawl } from './start-crawl.handler.js';
|
|
4
|
+
export { crawlRequestSchema } from './crawl-request.validator.js';
|
|
5
|
+
export type { CrawlRequest } from './crawl-request.validator.js';
|
|
6
|
+
export type { CrawlJob, CrawlResult, CrawlProgress, RecipeTask, RecipeTaskResult } from './crawl-job.model.js';
|
|
7
|
+
export { parseCrawlJob } from './crawl-job.validator.js';
|
|
8
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { HostSettings } from '../host-options/index.js';
|
|
2
|
+
import type { RecipeTask, RecipeTaskResult } from './crawl-job.model.js';
|
|
3
|
+
/**
|
|
4
|
+
* The `/crawl` activity: one input recipe, on a crawler of its own, limited
|
|
5
|
+
* to the caller's hosts. A recipe that fails is reported, not thrown: the
|
|
6
|
+
* orchestration still gathers the others.
|
|
7
|
+
*
|
|
8
|
+
* @param settings - The host settings.
|
|
9
|
+
* @param task - The recipe and where its records go.
|
|
10
|
+
* @returns Its reports (one per matrix variant) and records (or their link).
|
|
11
|
+
*/
|
|
12
|
+
export declare function runRecipeTask(settings: HostSettings, task: RecipeTask): Promise<RecipeTaskResult>;
|
|
13
|
+
//# sourceMappingURL=run-recipe.use-case.d.ts.map
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { HttpRequest, HttpResponse, HttpResponseInit } from '@azure/functions';
|
|
2
|
+
import type { TypedOrchestration } from '@mnci/az-durable';
|
|
3
|
+
import type { DurableClient } from 'durable-functions';
|
|
4
|
+
import type { HostSettings } from '../host-options/index.js';
|
|
5
|
+
import type { CrawlJob, CrawlResult } from './crawl-job.model.js';
|
|
6
|
+
/**
|
|
7
|
+
* `POST /crawl`: checks the caller and the recipes, then starts the
|
|
8
|
+
* orchestration. Recipes that do not load are a 400 before anything is
|
|
9
|
+
* queued.
|
|
10
|
+
*
|
|
11
|
+
* @param request - The request.
|
|
12
|
+
* @param client - The Durable client.
|
|
13
|
+
* @param settings - The host settings.
|
|
14
|
+
* @param orchestration - The `/crawl` orchestration.
|
|
15
|
+
* @returns 202 with the Durable status URLs, or the problem.
|
|
16
|
+
*/
|
|
17
|
+
export declare function startCrawl(request: HttpRequest, client: DurableClient, settings: HostSettings, orchestration: TypedOrchestration<CrawlJob, CrawlResult>): Promise<HttpResponse | HttpResponseInit>;
|
|
18
|
+
//# sourceMappingURL=start-crawl.handler.d.ts.map
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import type { HttpRequest } from '@azure/functions';
|
|
2
|
+
import type { AccessConfig, AccessPlugin, BrowserSessionConfig, CaptchaSolver, CrawlOptions, HookMap, WindowsPolicy } from '@opencraw/core';
|
|
3
|
+
import type { RecipeStore } from '../recipe-store/index.js';
|
|
4
|
+
import type { ResultStore } from '../result-store/index.js';
|
|
5
|
+
/** How the host is set up: everything that is code or a secret lives here, never in a request. */
|
|
6
|
+
export interface OpenCrawHostOptions {
|
|
7
|
+
/**
|
|
8
|
+
* The hosts a caller's recipes may reach (`example.com`, `*.example.com`,
|
|
9
|
+
* `host:port`, `*`). A list applies to every caller; a function decides per
|
|
10
|
+
* caller, and a caller it returns nothing for is refused (403).
|
|
11
|
+
*/
|
|
12
|
+
allowedHosts: readonly string[] | ((caller: string | undefined) => readonly string[] | undefined);
|
|
13
|
+
/**
|
|
14
|
+
* Who is calling, from the request: an Entra ID principal (App Service
|
|
15
|
+
* authentication sets `x-ms-client-principal-name`), a key name, a header
|
|
16
|
+
* your gateway sets. Only trust a header something in front of the app
|
|
17
|
+
* sets and callers cannot. Default: nobody in particular.
|
|
18
|
+
*/
|
|
19
|
+
identify?: (request: HttpRequest) => string | undefined;
|
|
20
|
+
/** Named recipes: what `/jobs` items and `/crawl` by name run. */
|
|
21
|
+
recipes?: RecipeStore;
|
|
22
|
+
/** Where results too large to return inline go. Without one, every result is returned inline. */
|
|
23
|
+
results?: ResultStore;
|
|
24
|
+
/** Results larger than this (as JSON) go to `results`. Default 256 KiB. */
|
|
25
|
+
inlineLimitBytes?: number;
|
|
26
|
+
hooks?: HookMap;
|
|
27
|
+
/** A fresh set per crawler: a crawler closes its solvers when it closes. */
|
|
28
|
+
captchaSolvers?: () => CaptchaSolver[];
|
|
29
|
+
access?: AccessConfig;
|
|
30
|
+
accessPlugins?: AccessPlugin[];
|
|
31
|
+
browser?: BrowserSessionConfig;
|
|
32
|
+
/** Worker pools: one per caller and crawl id. */
|
|
33
|
+
pools?: PoolSettings;
|
|
34
|
+
/**
|
|
35
|
+
* The recipe-authoring MCP endpoint, `/mcp`: `true`, or its sample limits.
|
|
36
|
+
* Off by default. It refuses to start without authentication.
|
|
37
|
+
*/
|
|
38
|
+
mcp?: boolean | McpSettings;
|
|
39
|
+
/** Who may promote a draft to production (`POST /recipes/{name}/{version}/promote`). Default: nobody. */
|
|
40
|
+
canPromote?: (caller: string | undefined) => boolean;
|
|
41
|
+
/** Default `function`: a function key is needed. `anonymous` only behind something that authenticates. */
|
|
42
|
+
authLevel?: 'anonymous' | 'function' | 'admin';
|
|
43
|
+
/** Prepended to every route: `'opencraw'` gives `/api/opencraw/crawl`. Default none. */
|
|
44
|
+
routePrefix?: string;
|
|
45
|
+
}
|
|
46
|
+
export interface McpSettings {
|
|
47
|
+
/** A sample run (`run` without `full`) stops after this many records per input recipe. Default 20. */
|
|
48
|
+
sampleRecords?: number;
|
|
49
|
+
/** And after this long, whatever it has by then. Default 60 seconds: MCP clients time out tool calls. */
|
|
50
|
+
sampleMs?: number;
|
|
51
|
+
}
|
|
52
|
+
export interface PoolSettings {
|
|
53
|
+
/** A pool with no item for this long closes. Default 10 minutes. */
|
|
54
|
+
idleTtlMs?: number;
|
|
55
|
+
/** The windows policy a job gets when its request names none. Default `{ min: 1, max: 4, grow: { after: 5 }, idle: { afterMs: 60000 } }`. */
|
|
56
|
+
windows?: WindowsPolicy;
|
|
57
|
+
/** No pool gets more windows than this, whatever a job asks. Default 8. */
|
|
58
|
+
maxWindows?: number;
|
|
59
|
+
}
|
|
60
|
+
/** The options with their defaults. */
|
|
61
|
+
export interface HostSettings extends Omit<OpenCrawHostOptions, 'pools' | 'inlineLimitBytes' | 'routePrefix' | 'authLevel' | 'mcp' | 'canPromote'> {
|
|
62
|
+
inlineLimitBytes: number;
|
|
63
|
+
routePrefix: string;
|
|
64
|
+
authLevel: 'anonymous' | 'function' | 'admin';
|
|
65
|
+
pools: Required<PoolSettings>;
|
|
66
|
+
/** `undefined` when the endpoint is off. */
|
|
67
|
+
mcp?: Required<McpSettings>;
|
|
68
|
+
canPromote: (caller: string | undefined) => boolean;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* @param options - As given.
|
|
72
|
+
* @returns The options with their defaults.
|
|
73
|
+
* @throws Error when `allowedHosts` is an empty list (a host that may reach nothing runs nothing), or the MCP endpoint would be open to anyone.
|
|
74
|
+
*/
|
|
75
|
+
export declare function resolveHostOptions(options: OpenCrawHostOptions): HostSettings;
|
|
76
|
+
/**
|
|
77
|
+
* What a crawler for one caller is created with: the host's code and
|
|
78
|
+
* secrets, and the caller's hosts.
|
|
79
|
+
*
|
|
80
|
+
* @param settings - The host settings.
|
|
81
|
+
* @param allowedHosts - The caller's hosts.
|
|
82
|
+
* @returns Crawl options, without a sink.
|
|
83
|
+
*/
|
|
84
|
+
export declare function crawlOptionsFor(settings: HostSettings, allowedHosts: readonly string[]): CrawlOptions;
|
|
85
|
+
//# sourceMappingURL=host-options.config.d.ts.map
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { HttpRequest, HttpResponseInit } from '@azure/functions';
|
|
2
|
+
/**
|
|
3
|
+
* @param status - The HTTP status.
|
|
4
|
+
* @param body - A JSON body.
|
|
5
|
+
* @returns The response.
|
|
6
|
+
*/
|
|
7
|
+
export declare function reply(status: number, body: unknown): HttpResponseInit;
|
|
8
|
+
/**
|
|
9
|
+
* An error the caller can act on: what went wrong, and the issues a recipe
|
|
10
|
+
* error lists (each with its JSON path).
|
|
11
|
+
*
|
|
12
|
+
* @param status - The HTTP status.
|
|
13
|
+
* @param error - What went wrong.
|
|
14
|
+
* @returns The response.
|
|
15
|
+
*/
|
|
16
|
+
export declare function problem(status: number, error: unknown): HttpResponseInit;
|
|
17
|
+
/**
|
|
18
|
+
* The request body as JSON.
|
|
19
|
+
*
|
|
20
|
+
* @param request - The request.
|
|
21
|
+
* @returns The body, or `undefined` when it is not JSON.
|
|
22
|
+
*/
|
|
23
|
+
export declare function readJson(request: HttpRequest): Promise<unknown>;
|
|
24
|
+
//# sourceMappingURL=http-reply.mapper.d.ts.map
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export { registerOpenCraw, hostJsonWarnings } from './open-craw-host/index.js';
|
|
2
|
+
export type { OpenCrawHost } from './open-craw-host/index.js';
|
|
3
|
+
export type { OpenCrawHostOptions, PoolSettings, McpSettings, HostSettings } from './host-options/index.js';
|
|
4
|
+
export { memoryRecipes, blobRecipes, RecipeStoreError } from './recipe-store/index.js';
|
|
5
|
+
export type { RecipeStore, StoredRecipes, BlobRecipesOptions } from './recipe-store/index.js';
|
|
6
|
+
export { blobResults } from './result-store/index.js';
|
|
7
|
+
export type { ResultStore, BlobResultsOptions } from './result-store/index.js';
|
|
8
|
+
export type { CrawlJob, CrawlResult, CrawlProgress, CrawlRequest, RecipeTask, RecipeTaskResult } from './crawl-run/index.js';
|
|
9
|
+
export type { ItemJob, ItemOutcome, PoolState, ItemRequest } from './worker-pools/index.js';
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What in the Function App's `host.json` would make the host misbehave. An
|
|
3
|
+
* item waiting in a pool's inbox holds an activity slot, so a pool can only
|
|
4
|
+
* fill its windows if Durable runs that many activities at once.
|
|
5
|
+
*
|
|
6
|
+
* @param maxWindows - The most windows one pool may have.
|
|
7
|
+
* @param directory - Where `host.json` is; the working directory by default.
|
|
8
|
+
* @returns Warnings, none when `host.json` is missing or fine.
|
|
9
|
+
*/
|
|
10
|
+
export declare function hostJsonWarnings(maxWindows: number, directory?: string): string[];
|
|
11
|
+
//# sourceMappingURL=host-json.validator.d.ts.map
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { HttpRequest, HttpResponse, HttpResponseInit } from '@azure/functions';
|
|
2
|
+
import type { TypedOrchestration } from '@mnci/az-durable';
|
|
3
|
+
import type { DurableClient } from 'durable-functions';
|
|
4
|
+
import type { CrawlJob, CrawlResult, RecipeTask, RecipeTaskResult } from '../crawl-run/index.js';
|
|
5
|
+
import type { HostSettings, OpenCrawHostOptions } from '../host-options/index.js';
|
|
6
|
+
import { WorkerPools } from '../worker-pools/index.js';
|
|
7
|
+
import type { ItemJob, ItemOutcome } from '../worker-pools/index.js';
|
|
8
|
+
/** The functions the host registered, for tests and for an app that wants to call them itself. */
|
|
9
|
+
export interface OpenCrawHost {
|
|
10
|
+
settings: HostSettings;
|
|
11
|
+
pools: WorkerPools;
|
|
12
|
+
orchestrations: {
|
|
13
|
+
crawl: TypedOrchestration<CrawlJob, CrawlResult>;
|
|
14
|
+
item: TypedOrchestration<ItemJob, ItemOutcome>;
|
|
15
|
+
};
|
|
16
|
+
activities: {
|
|
17
|
+
runRecipe: (task: RecipeTask) => Promise<RecipeTaskResult>;
|
|
18
|
+
runItem: (job: ItemJob) => Promise<ItemOutcome>;
|
|
19
|
+
};
|
|
20
|
+
http: {
|
|
21
|
+
startCrawl: (request: HttpRequest, client: DurableClient) => Promise<HttpResponse | HttpResponseInit>;
|
|
22
|
+
submitItem: (request: HttpRequest, client: DurableClient) => Promise<HttpResponse | HttpResponseInit>;
|
|
23
|
+
job: (request: HttpRequest) => Promise<HttpResponseInit>;
|
|
24
|
+
/** The authoring MCP endpoint; registered only when `mcp` is on. */
|
|
25
|
+
mcp: (request: HttpRequest, client: DurableClient) => Promise<HttpResponseInit>;
|
|
26
|
+
promote: (request: HttpRequest) => Promise<HttpResponseInit>;
|
|
27
|
+
};
|
|
28
|
+
/** Closes every pool; also run when the Function App shuts down. */
|
|
29
|
+
close: () => Promise<void>;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Registers OpenCraw's HTTP API in this Function App (the v4 programming
|
|
33
|
+
* model): `POST /crawl` runs recipes once, `POST /jobs/{crawlId}/items`
|
|
34
|
+
* sends one item to the caller's warm pool, `GET` and `DELETE
|
|
35
|
+
* /jobs/{crawlId}` read and close it, `POST /recipes/{name}/{version}/promote`
|
|
36
|
+
* makes a draft runnable, and with `mcp` on, `/mcp` serves the authoring
|
|
37
|
+
* tools. Call it once, from the app's entry module; nothing is registered by
|
|
38
|
+
* importing the package.
|
|
39
|
+
*
|
|
40
|
+
* @param options - Allowed hosts, callers, recipes, result storage, plugins, pools.
|
|
41
|
+
* @returns The registered pieces.
|
|
42
|
+
* @throws Error when the options cannot work, or when it is called twice (function names are global).
|
|
43
|
+
*/
|
|
44
|
+
export declare function registerOpenCraw(options: OpenCrawHostOptions): OpenCrawHost;
|
|
45
|
+
//# sourceMappingURL=register-open-craw.use-case.d.ts.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
2
|
+
import type { AuthoringContext } from './authoring-tools.use-case.js';
|
|
3
|
+
/**
|
|
4
|
+
* The authoring MCP server for one call: the tools an agent writes recipes
|
|
5
|
+
* with, run where the recipes will run, for this caller only. Unlike the
|
|
6
|
+
* local `@opencraw/mcp`, nothing takes a path: recipes travel inline, and a
|
|
7
|
+
* finished set is published to the host's store.
|
|
8
|
+
*
|
|
9
|
+
* @param context - The host, the caller, Durable.
|
|
10
|
+
* @returns The server, ready to `connect`.
|
|
11
|
+
*/
|
|
12
|
+
export declare function createAuthoringServer(context: AuthoringContext): McpServer;
|
|
13
|
+
//# sourceMappingURL=authoring-server.use-case.d.ts.map
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
|
2
|
+
import type { TypedOrchestration } from '@mnci/az-durable';
|
|
3
|
+
import type { DurableClient } from 'durable-functions';
|
|
4
|
+
import type { CallerAccess } from '../caller-access/index.js';
|
|
5
|
+
import type { CrawlJob, CrawlResult } from '../crawl-run/index.js';
|
|
6
|
+
import type { HostSettings } from '../host-options/index.js';
|
|
7
|
+
/** What every authoring tool call works with: the host, the caller, and Durable for full runs. */
|
|
8
|
+
export interface AuthoringContext {
|
|
9
|
+
settings: HostSettings & {
|
|
10
|
+
mcp: NonNullable<HostSettings['mcp']>;
|
|
11
|
+
};
|
|
12
|
+
access: CallerAccess;
|
|
13
|
+
client: DurableClient;
|
|
14
|
+
crawl: TypedOrchestration<CrawlJob, CrawlResult>;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* `probe`: where a page's data lives, fetched from the host itself (its IP,
|
|
18
|
+
* its proxies) and only from the caller's hosts.
|
|
19
|
+
*
|
|
20
|
+
* @param context - The call's context.
|
|
21
|
+
* @param args - The page, whether to render it, an access profile.
|
|
22
|
+
* @returns The findings.
|
|
23
|
+
*/
|
|
24
|
+
export declare function probe(context: AuthoringContext, args: {
|
|
25
|
+
url: string;
|
|
26
|
+
browser?: boolean;
|
|
27
|
+
access?: string;
|
|
28
|
+
}): Promise<CallToolResult>;
|
|
29
|
+
/**
|
|
30
|
+
* `validate`: `@opencraw/mcp`'s own, on inline recipes only (never paths on the host).
|
|
31
|
+
*
|
|
32
|
+
* @param args - The recipes.
|
|
33
|
+
* @returns Every issue with its JSON path.
|
|
34
|
+
*/
|
|
35
|
+
export declare function validate(args: {
|
|
36
|
+
recipes: string | Record<string, unknown>[];
|
|
37
|
+
}): Promise<CallToolResult>;
|
|
38
|
+
/**
|
|
39
|
+
* `run`: a sample by default (a few records per input recipe, within the
|
|
40
|
+
* time a tool call may take), or with `full` a Durable crawl whose instance
|
|
41
|
+
* id `status` reads. Drafts run here; production routes refuse them.
|
|
42
|
+
*
|
|
43
|
+
* @param context - The call's context.
|
|
44
|
+
* @param args - Inline recipes or a stored set, and whether to run it all.
|
|
45
|
+
* @returns The sample's reports and records, or the instance id.
|
|
46
|
+
*/
|
|
47
|
+
export declare function run(context: AuthoringContext, args: {
|
|
48
|
+
recipes?: Record<string, unknown>[];
|
|
49
|
+
recipe?: {
|
|
50
|
+
name: string;
|
|
51
|
+
version: string;
|
|
52
|
+
};
|
|
53
|
+
full?: boolean;
|
|
54
|
+
}): Promise<CallToolResult>;
|
|
55
|
+
/**
|
|
56
|
+
* `status`: a full run's state and result, for the caller who started it.
|
|
57
|
+
*
|
|
58
|
+
* @param context - The call's context.
|
|
59
|
+
* @param args - The instance id `run` returned.
|
|
60
|
+
* @returns Its runtime status, progress and output.
|
|
61
|
+
*/
|
|
62
|
+
export declare function status(context: AuthoringContext, args: {
|
|
63
|
+
instanceId: string;
|
|
64
|
+
}): Promise<CallToolResult>;
|
|
65
|
+
/**
|
|
66
|
+
* `list_recipes`: the stored recipe sets, with their versions and states.
|
|
67
|
+
*
|
|
68
|
+
* @param context - The call's context.
|
|
69
|
+
* @returns The list.
|
|
70
|
+
*/
|
|
71
|
+
export declare function listRecipes(context: AuthoringContext): Promise<CallToolResult>;
|
|
72
|
+
/**
|
|
73
|
+
* `get_recipes`: a stored set's recipes, drafts included: the start of a new version.
|
|
74
|
+
*
|
|
75
|
+
* @param context - The call's context.
|
|
76
|
+
* @param args - The name and version.
|
|
77
|
+
* @returns The set.
|
|
78
|
+
*/
|
|
79
|
+
export declare function getRecipes(context: AuthoringContext, args: {
|
|
80
|
+
name: string;
|
|
81
|
+
version: string;
|
|
82
|
+
}): Promise<CallToolResult>;
|
|
83
|
+
/**
|
|
84
|
+
* `publish`: saves recipes that load as a new version, as a draft. A person
|
|
85
|
+
* (or a caller allowed to) promotes it before production runs it.
|
|
86
|
+
*
|
|
87
|
+
* @param context - The call's context.
|
|
88
|
+
* @param args - The name, a new version, the recipes.
|
|
89
|
+
* @returns The stored version.
|
|
90
|
+
*/
|
|
91
|
+
export declare function publish(context: AuthoringContext, args: {
|
|
92
|
+
name: string;
|
|
93
|
+
version: string;
|
|
94
|
+
recipes: Record<string, unknown>[];
|
|
95
|
+
}): Promise<CallToolResult>;
|
|
96
|
+
//# sourceMappingURL=authoring-tools.use-case.d.ts.map
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { mcpEndpoint } from './mcp-endpoint.handler.js';
|
|
2
|
+
export { promoteRecipes } from './promote.handler.js';
|
|
3
|
+
export { createAuthoringServer } from './authoring-server.use-case.js';
|
|
4
|
+
export type { AuthoringContext } from './authoring-tools.use-case.js';
|
|
5
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { HttpRequest, HttpResponseInit } from '@azure/functions';
|
|
2
|
+
import type { TypedOrchestration } from '@mnci/az-durable';
|
|
3
|
+
import type { DurableClient } from 'durable-functions';
|
|
4
|
+
import type { CrawlJob, CrawlResult } from '../crawl-run/index.js';
|
|
5
|
+
import type { HostSettings } from '../host-options/index.js';
|
|
6
|
+
/**
|
|
7
|
+
* `/mcp`: the authoring MCP server over Streamable HTTP, stateless: every
|
|
8
|
+
* request gets its own server and transport, and a JSON answer, so any
|
|
9
|
+
* instance can take any call.
|
|
10
|
+
*
|
|
11
|
+
* @param request - The request.
|
|
12
|
+
* @param client - The Durable client, for full runs.
|
|
13
|
+
* @param settings - The host settings.
|
|
14
|
+
* @param crawl - The `/crawl` orchestration.
|
|
15
|
+
* @returns The MCP answer, 403 for a caller with no hosts, 404 when the endpoint is off.
|
|
16
|
+
*/
|
|
17
|
+
export declare function mcpEndpoint(request: HttpRequest, client: DurableClient, settings: HostSettings, crawl: TypedOrchestration<CrawlJob, CrawlResult>): Promise<HttpResponseInit>;
|
|
18
|
+
//# sourceMappingURL=mcp-endpoint.handler.d.ts.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { HttpRequest, HttpResponseInit } from '@azure/functions';
|
|
2
|
+
import type { HostSettings } from '../host-options/index.js';
|
|
3
|
+
/**
|
|
4
|
+
* `POST /recipes/{name}/{version}/promote`: a draft becomes runnable in
|
|
5
|
+
* production. Only for callers `canPromote` allows: an agent publishes, a
|
|
6
|
+
* person promotes.
|
|
7
|
+
*
|
|
8
|
+
* @param request - The request.
|
|
9
|
+
* @param settings - The host settings.
|
|
10
|
+
* @returns The promoted version, 403, or the store's 404.
|
|
11
|
+
*/
|
|
12
|
+
export declare function promoteRecipes(request: HttpRequest, settings: HostSettings): Promise<HttpResponseInit>;
|
|
13
|
+
//# sourceMappingURL=promote.handler.d.ts.map
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { RecipeStore, StoredRecipes } from './recipe-store.contract.js';
|
|
2
|
+
export interface BlobRecipesOptions {
|
|
3
|
+
/** A storage connection string. */
|
|
4
|
+
connectionString: string;
|
|
5
|
+
/** Default `opencraw-recipes`; created when missing. */
|
|
6
|
+
container?: string;
|
|
7
|
+
/** The recipe sets the deployment ships with: read-only, promoted, and never shadowed by a publish. */
|
|
8
|
+
shipped?: Omit<StoredRecipes, 'state'>[];
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Named recipes in Azure Blob Storage: one JSON blob per version, its state
|
|
12
|
+
* (`draft` or `promoted`) in the blob's metadata. A version is written once:
|
|
13
|
+
* a second publish of it fails even when two race, since the write only
|
|
14
|
+
* succeeds when no blob is there.
|
|
15
|
+
*
|
|
16
|
+
* @param options - The account, the container, and the sets the deployment ships with.
|
|
17
|
+
* @returns The store.
|
|
18
|
+
*/
|
|
19
|
+
export declare function blobRecipes(options: BlobRecipesOptions): RecipeStore;
|
|
20
|
+
//# sourceMappingURL=blob-recipe.store.d.ts.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { RecipeStore, StoredRecipes } from './recipe-store.contract.js';
|
|
2
|
+
/**
|
|
3
|
+
* A stored recipe set production may run.
|
|
4
|
+
*
|
|
5
|
+
* @param store - The store, if the host has one.
|
|
6
|
+
* @param name - The set's name.
|
|
7
|
+
* @param version - Its version.
|
|
8
|
+
* @param allowDraft - Whether a draft will do (authoring tools), or only a promoted version (production).
|
|
9
|
+
* @returns The set.
|
|
10
|
+
* @throws RecipeStoreError 404 when there is no such version, 409 when it is a draft and a draft won't do.
|
|
11
|
+
*/
|
|
12
|
+
export declare function findRecipes(store: RecipeStore | undefined, name: string, version: string, allowDraft?: boolean): Promise<StoredRecipes>;
|
|
13
|
+
//# sourceMappingURL=find-recipes.use-case.d.ts.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { RecipeStoreError } from './recipe-store.error.js';
|
|
2
|
+
export type { RecipeStore, StoredRecipes } from './recipe-store.contract.js';
|
|
3
|
+
export { memoryRecipes } from './memory-recipe.store.js';
|
|
4
|
+
export { findRecipes } from './find-recipes.use-case.js';
|
|
5
|
+
export { blobRecipes } from './blob-recipe.store.js';
|
|
6
|
+
export type { BlobRecipesOptions } from './blob-recipe.store.js';
|
|
7
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { RecipeStore, StoredRecipes } from './recipe-store.contract.js';
|
|
2
|
+
/**
|
|
3
|
+
* Recipes held in memory: the ones a deployment ships with (promoted), plus
|
|
4
|
+
* drafts published while it runs, which a restart forgets.
|
|
5
|
+
*
|
|
6
|
+
* @param entries - The recipe sets; `state` defaults to `promoted`.
|
|
7
|
+
* @returns The store.
|
|
8
|
+
*/
|
|
9
|
+
export declare function memoryRecipes(entries?: (Omit<StoredRecipes, 'state'> & {
|
|
10
|
+
state?: StoredRecipes['state'];
|
|
11
|
+
})[]): RecipeStore;
|
|
12
|
+
//# sourceMappingURL=memory-recipe.store.d.ts.map
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/** A named, versioned recipe set: its output recipe and its input recipes. */
|
|
2
|
+
export interface StoredRecipes {
|
|
3
|
+
name: string;
|
|
4
|
+
/** Any string: `3`, `2026-09-27`. A version, once written, never changes. */
|
|
5
|
+
version: string;
|
|
6
|
+
/** The recipes as JSON objects: one output recipe and its input recipes, in any order. */
|
|
7
|
+
recipes: unknown[];
|
|
8
|
+
/** A draft can be tried, but production routes (`/crawl` by name, `/jobs`) refuse it. */
|
|
9
|
+
state: 'draft' | 'promoted';
|
|
10
|
+
}
|
|
11
|
+
/** Where the host finds named recipes. */
|
|
12
|
+
export interface RecipeStore {
|
|
13
|
+
get(name: string, version: string): Promise<StoredRecipes | undefined>;
|
|
14
|
+
list(): Promise<Omit<StoredRecipes, 'recipes'>[]>;
|
|
15
|
+
/**
|
|
16
|
+
* Saves a new version as a draft.
|
|
17
|
+
*
|
|
18
|
+
* @throws Error when that version exists: versions never change.
|
|
19
|
+
*/
|
|
20
|
+
put(entry: Omit<StoredRecipes, 'state'>): Promise<void>;
|
|
21
|
+
/**
|
|
22
|
+
* Makes a draft runnable in production.
|
|
23
|
+
*
|
|
24
|
+
* @throws Error when there is no such version.
|
|
25
|
+
*/
|
|
26
|
+
promote(name: string, version: string): Promise<void>;
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=recipe-store.contract.d.ts.map
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { ResultStore } from './result-store.contract.js';
|
|
2
|
+
export interface BlobResultsOptions {
|
|
3
|
+
/** A storage connection string with an account key (the key signs the read links). */
|
|
4
|
+
connectionString: string;
|
|
5
|
+
/** Default `opencraw-results`; created when missing. */
|
|
6
|
+
container?: string;
|
|
7
|
+
/** How long a read link works. Default 24 hours. */
|
|
8
|
+
linkTtlMs?: number;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Results in Azure Blob Storage, as JSON Lines, returned as read-only SAS
|
|
12
|
+
* links that expire.
|
|
13
|
+
*
|
|
14
|
+
* @param options - The account, container and link lifetime.
|
|
15
|
+
* @returns The store.
|
|
16
|
+
*/
|
|
17
|
+
export declare function blobResults(options: BlobResultsOptions): ResultStore;
|
|
18
|
+
//# sourceMappingURL=blob-result.store.d.ts.map
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export type { ResultStore } from './result-store.contract.js';
|
|
2
|
+
export { blobResults } from './blob-result.store.js';
|
|
3
|
+
export type { BlobResultsOptions } from './blob-result.store.js';
|
|
4
|
+
export { packRecords } from './result-packing.policy.js';
|
|
5
|
+
export type { PackedRecords } from './result-packing.policy.js';
|
|
6
|
+
//# sourceMappingURL=index.d.ts.map
|