@anchrd/colab-sdk 0.1.19 → 0.1.21

package/dist/crawls/crawls.d.ts

@@ -0,0 +1,132 @@
+import { z } from "zod";
+export declare const CrawlId: z.ZodTemplateLiteral<`crawl_${string}`>;
+export type CrawlId = z.infer<typeof CrawlId>;
+export declare const maxCrawlPages = 1000;
+export declare const defaultCrawlPages = 100;
+export declare const maxCrawlPagesPerDay = 20000;
+export declare const crawlDays = 14;
+export declare const maxCrawlHits = 20;
+export declare const maxCrawlReads = 10;
+export declare const crawlPassageChars = 300;
+export declare const CrawlCreateInput: z.ZodObject<{
+    url: z.ZodString;
+    includePatterns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    limit: z.ZodDefault<z.ZodNumber>;
+    depth: z.ZodOptional<z.ZodNumber>;
+    render: z.ZodDefault<z.ZodBoolean>;
+    source: z.ZodDefault<z.ZodEnum<{
+        links: "links";
+        all: "all";
+        sitemaps: "sitemaps";
+    }>>;
+    modifiedSince: z.ZodOptional<z.ZodISODateTime>;
+    idempotencyKey: z.ZodString;
+}, z.core.$strict>;
+export type CrawlCreateInput = z.infer<typeof CrawlCreateInput>;
+export declare const CrawlGetInput: z.ZodObject<{
+    crawlId: z.ZodTemplateLiteral<`crawl_${string}`>;
+    query: z.ZodOptional<z.ZodString>;
+    urls: z.ZodOptional<z.ZodPreprocess<z.ZodArray<z.ZodString>, unknown>>;
+    cursor: z.ZodOptional<z.ZodString>;
+    limit: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
+    errors: z.ZodOptional<z.ZodPreprocess<z.ZodBoolean, unknown>>;
+}, z.core.$strict>;
+export type CrawlGetInput = z.infer<typeof CrawlGetInput>;
+export declare const CrawlStatus: z.ZodEnum<{
+    done: "done";
+    failed: "failed";
+    expired: "expired";
+    running: "running";
+    starting: "starting";
+    reading: "reading";
+}>;
+export type CrawlStatus = z.infer<typeof CrawlStatus>;
+export declare const Crawl: z.ZodObject<{
+    crawlId: z.ZodTemplateLiteral<`crawl_${string}`>;
+    url: z.ZodString;
+    status: z.ZodEnum<{
+        done: "done";
+        failed: "failed";
+        expired: "expired";
+        running: "running";
+        starting: "starting";
+        reading: "reading";
+    }>;
+    reason: z.ZodNullable<z.ZodString>;
+    limit: z.ZodNumber;
+    render: z.ZodBoolean;
+    total: z.ZodNullable<z.ZodNumber>;
+    finished: z.ZodNullable<z.ZodNumber>;
+    pages: z.ZodNumber;
+    createdAt: z.ZodISODateTime;
+    expiresAt: z.ZodISODateTime;
+}, z.core.$strip>;
+export type Crawl = z.infer<typeof Crawl>;
+export declare const CrawlPage: z.ZodObject<{
+    url: z.ZodString;
+    title: z.ZodString;
+    length: z.ZodNumber;
+}, z.core.$strip>;
+export type CrawlPage = z.infer<typeof CrawlPage>;
+export declare const CrawlHit: z.ZodObject<{
+    url: z.ZodString;
+    title: z.ZodString;
+    passage: z.ZodString;
+}, z.core.$strip>;
+export type CrawlHit = z.infer<typeof CrawlHit>;
+export declare const CrawlRead: z.ZodObject<{
+    url: z.ZodString;
+    title: z.ZodString;
+    markdown: z.ZodString;
+}, z.core.$strip>;
+export type CrawlRead = z.infer<typeof CrawlRead>;
+export declare const CrawlFailure: z.ZodObject<{
+    url: z.ZodString;
+    status: z.ZodString;
+}, z.core.$strip>;
+export type CrawlFailure = z.infer<typeof CrawlFailure>;
+export declare const CrawlDetail: z.ZodObject<{
+    crawl: z.ZodObject<{
+        crawlId: z.ZodTemplateLiteral<`crawl_${string}`>;
+        url: z.ZodString;
+        status: z.ZodEnum<{
+            done: "done";
+            failed: "failed";
+            expired: "expired";
+            running: "running";
+            starting: "starting";
+            reading: "reading";
+        }>;
+        reason: z.ZodNullable<z.ZodString>;
+        limit: z.ZodNumber;
+        render: z.ZodBoolean;
+        total: z.ZodNullable<z.ZodNumber>;
+        finished: z.ZodNullable<z.ZodNumber>;
+        pages: z.ZodNumber;
+        createdAt: z.ZodISODateTime;
+        expiresAt: z.ZodISODateTime;
+    }, z.core.$strip>;
+    items: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        url: z.ZodString;
+        title: z.ZodString;
+        length: z.ZodNumber;
+    }, z.core.$strip>>>;
+    nextCursor: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+    hits: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        url: z.ZodString;
+        title: z.ZodString;
+        passage: z.ZodString;
+    }, z.core.$strip>>>;
+    reads: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        url: z.ZodString;
+        title: z.ZodString;
+        markdown: z.ZodString;
+    }, z.core.$strip>>>;
+    missing: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    failures: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        url: z.ZodString;
+        status: z.ZodString;
+    }, z.core.$strip>>>;
+}, z.core.$strip>;
+export type CrawlDetail = z.infer<typeof CrawlDetail>;

package/dist/crawls/crawls.js

@@ -0,0 +1,138 @@
+import { z } from "zod";
+import { idSchema } from "../ids/ids.js";
+import { Flag, Limit as PageSize } from "../list-input.js";
+import { IdempotencyKey } from "../primitives/primitives.js";
+// Crawls (#785, E137, Funktionen › Crawl): the inputs of `intelligence_crawl_create` and
+// `intelligence_crawl_get`, and their answers. Colab starts a crawl of a known website at Cloudflare
+// Browser Rendering (`/crawl`), keeps its pages 14 days in a search table and stores none of them as
+// a node: whoever wants to keep a page makes it a file with `intelligence_file_create`.
+export const CrawlId = idSchema("crawl").describe("The crawl's id, `crawl_…`, as intelligence_crawl_create answered it.");
+// The most pages one crawl may order, the default, and the most a Colab orders in one day (UTC).
+export const maxCrawlPages = 1000;
+export const defaultCrawlPages = 100;
+export const maxCrawlPagesPerDay = 20_000;
+// How long a crawl's pages stay searchable, like Cloudflare keeps them.
+export const crawlDays = 14;
+// The most hits a search answers, the most pages read whole in one call, and the length of a hit's
+// passage.
+export const maxCrawlHits = 20;
+export const maxCrawlReads = 10;
+export const crawlPassageChars = 300;
+const Pattern = z.string().min(1).max(500);
+export const CrawlCreateInput = z.strictObject({
+    url: z
+        .string()
+        .min(1)
+        .max(2000)
+        .describe("Where the crawl starts: a full https address of a public website, such as https://www.bitkom.org/Presse. Cloudflare follows its links and its sitemap from there."),
+    includePatterns: z
+        .array(Pattern)
+        .max(20)
+        .optional()
+        .describe("Only addresses matching one of these are crawled, such as https://www.bitkom.org/Presse/**. `*` matches anything but `/`, `**` anything. Leaving it out takes every address the crawl finds on the site."),
+    excludePatterns: z
+        .array(Pattern)
+        .max(20)
+        .optional()
+        .describe("Addresses matching one of these are left out; they win over includePatterns."),
+    limit: z
+        .number()
+        .int()
+        .min(1)
+        .max(maxCrawlPages, {
+        error: `A crawl orders at most ${maxCrawlPages} pages. Narrow the site with includePatterns or start below it.`,
+    })
+        .default(defaultCrawlPages)
+        .describe(`How many pages to crawl at most, 1 to ${maxCrawlPages}. Leaving it out gives ${defaultCrawlPages}. Start small, read the hits, and widen only when the area carries. A single page is a crawl with limit 1.`),
+    depth: z
+        .number()
+        .int()
+        .min(0)
+        .max(100_000)
+        .optional()
+        .describe("How many links deep from the start address, 0 for the start page alone. Leaving it out lets the limit decide."),
+    render: z
+        .boolean()
+        .default(false)
+        .describe("true runs a real browser and executes JavaScript, slower and billed by browser time; only for a page that stays empty without it. Leaving it out fetches the plain HTML."),
+    source: z
+        .enum(["all", "sitemaps", "links"])
+        .default("all")
+        .describe("Where the crawl finds pages: all (sitemap and links, the default), sitemaps or links only."),
+    modifiedSince: z.iso
+        .datetime()
+        .optional()
+        .describe("Only pages changed after this moment, ISO 8601. Set it to the last crawl of the same site for a recurring look, so only what is new comes back."),
+    idempotencyKey: IdempotencyKey,
+});
+export const CrawlGetInput = z.strictObject({
+    crawlId: CrawlId,
+    query: z
+        .string()
+        .min(2)
+        .max(200)
+        .optional()
+        .describe(`Search the crawl's pages instead of listing them: words to find in title and text, at least two characters; each word also finds longer words it begins. Answers the best ${maxCrawlHits} hits, each with one passage. Ask several times with other words instead of reading every page.`),
+    // Over REST one value, the addresses separated by spaces (`?urls=a%20b`): a query string carries
+    // text, and an address holds commas but never a bare space (#785, measured on colab.anchrd.sh).
+    urls: z
+        .preprocess((value) => (typeof value === "string" ? value.split(/\s+/).filter(Boolean) : value), z.array(z.string().min(1).max(2000)).min(1).max(maxCrawlReads))
+        .optional()
+        .describe(`Read these pages of the crawl whole, as Markdown, 1 to ${maxCrawlReads}, by the address the list or a hit named; over REST one value with the addresses separated by spaces. Not together with query.`),
+    cursor: z
+        .string()
+        .max(200)
+        .optional()
+        .describe("The nextCursor of the previous page of the list, unchanged."),
+    limit: PageSize.optional().describe("How many pages one page of the list holds, 1 to 100. Leaving it out gives 100."),
+    errors: Flag.optional().describe("true lists the addresses the crawl could not fetch or was not allowed to, with their reason, instead of the pages."),
+});
+// `starting` until Cloudflare took the job, `running` while it crawls, `reading` while Colab takes
+// its pages, `done` when they are searchable, `failed` when the job ended without pages (`reason`
+// says why), `expired` after `crawlDays`.
+export const CrawlStatus = z.enum(["starting", "running", "reading", "done", "failed", "expired"]);
+export const Crawl = z.object({
+    crawlId: CrawlId,
+    url: z.string(),
+    status: CrawlStatus,
+    reason: z.string().nullable(),
+    limit: z.number(),
+    render: z.boolean(),
+    // Pages Cloudflare found and finished so far, and the pages Colab holds once it is done.
+    total: z.number().nullable(),
+    finished: z.number().nullable(),
+    pages: z.number(),
+    createdAt: z.iso.datetime(),
+    expiresAt: z.iso.datetime(),
+});
+export const CrawlPage = z.object({
+    url: z.string(),
+    title: z.string(),
+    // Characters of the page's Markdown.
+    length: z.number(),
+});
+export const CrawlHit = z.object({
+    url: z.string(),
+    title: z.string(),
+    passage: z.string(),
+});
+export const CrawlRead = z.object({
+    url: z.string(),
+    title: z.string(),
+    markdown: z.string(),
+});
+export const CrawlFailure = z.object({
+    url: z.string(),
+    status: z.string(),
+});
+// One answer, with the part the call asked for: `items` (the list), `hits` (a search), `reads` (pages
+// whole) or `failures`. While the crawl is not done, only `crawl` comes back.
+export const CrawlDetail = z.object({
+    crawl: Crawl,
+    items: z.array(CrawlPage).optional(),
+    nextCursor: z.string().nullable().optional(),
+    hits: z.array(CrawlHit).optional(),
+    reads: z.array(CrawlRead).optional(),
+    missing: z.array(z.string()).optional(),
+    failures: z.array(CrawlFailure).optional(),
+});

package/dist/grammar/grammar.d.ts

@@ -33,6 +33,7 @@ export declare const resources: {
     readonly sso: "sso";
     readonly secret: "secrets";
     readonly step: "steps";
+    readonly crawl: "crawls";
 };
 export declare const actions: readonly ["list", "get", "create", "update", "archive", "restore", "purge"];
 export declare const verbs: readonly ["invite", "resend", "disable", "enable", "rotate", "share", "move", "start", "complete", "test", "revoke", "remove", "unlock", "publish", "schedule", "unpublish", "cancel"];

package/dist/grammar/grammar.js

@@ -60,6 +60,9 @@ export const resources = {
     // One step of an Operator run as its CLI's output showed it (E112), a sub-resource of the run:
     // `operator_run_step_list`. Not `event`, the protocol's resource, and not `message`, a chat's.
     step: "steps",
+    // A crawl of a known website at Cloudflare Browser Rendering (#785, E137): `intelligence_crawl_*`.
+    // Its pages are searched in Colab for 14 days and stored nowhere as a node.
+    crawl: "crawls",
 };
 // The closed list of actions with a route of their own shape.
 export const actions = ["list", "get", "create", "update", "archive", "restore", "purge"];

package/dist/sdk.d.ts

@@ -9,6 +9,7 @@ export { CardField, CardFieldId, CardFieldInput, CardFieldOption, CardFieldType,
 export { ColumnId, ColumnType, columnTypes, TaskColumn } from "./columns/columns.js";
 export { Connection, ConnectionCreateInput, ConnectionCredential, ConnectionDetail, ConnectionGetInput, ConnectionId, ConnectionList, ConnectionListInput, ConnectionRevokeInput, ConnectionState, ConnectionTested, ConnectionTestInput, ConnectionTestResult, ConnectionUpdateInput, } from "./connections/connections.js";
 export { Call, CallId, CallList, CallListInput, CallSource, Connector, ConnectorField, ConnectorGetInput, ConnectorId, ConnectorList, ConnectorListInput, ConnectorTool, } from "./connectors/connectors.js";
+export { Crawl, CrawlCreateInput, CrawlDetail, CrawlFailure, CrawlGetInput, CrawlHit, CrawlId, CrawlPage, CrawlRead, CrawlStatus, crawlDays, crawlPassageChars, defaultCrawlPages, maxCrawlHits, maxCrawlPages, maxCrawlPagesPerDay, maxCrawlReads, } from "./crawls/crawls.js";
 export { DirectoryPrincipal, DirectorySearchInput } from "./directory/directory.js";
 export { DocumentCreateInput, DocumentDetail, DocumentExcerpt, DocumentGetInput, DocumentUpdateInput, } from "./documents/documents.js";
 export { ErrorCode, ErrorCurrent, ErrorIssue, ErrorResponse, errorCodes, invalidInput, Warning, } from "./error/error.js";

package/dist/sdk.js

@@ -12,6 +12,7 @@ export { CardField, CardFieldId, CardFieldInput, CardFieldOption, CardFieldType,
 export { ColumnId, ColumnType, columnTypes, TaskColumn } from "./columns/columns.js";
 export { Connection, ConnectionCreateInput, ConnectionCredential, ConnectionDetail, ConnectionGetInput, ConnectionId, ConnectionList, ConnectionListInput, ConnectionRevokeInput, ConnectionState, ConnectionTested, ConnectionTestInput, ConnectionTestResult, ConnectionUpdateInput, } from "./connections/connections.js";
 export { Call, CallId, CallList, CallListInput, CallSource, Connector, ConnectorField, ConnectorGetInput, ConnectorId, ConnectorList, ConnectorListInput, ConnectorTool, } from "./connectors/connectors.js";
+export { Crawl, CrawlCreateInput, CrawlDetail, CrawlFailure, CrawlGetInput, CrawlHit, CrawlId, CrawlPage, CrawlRead, CrawlStatus, crawlDays, crawlPassageChars, defaultCrawlPages, maxCrawlHits, maxCrawlPages, maxCrawlPagesPerDay, maxCrawlReads, } from "./crawls/crawls.js";
 export { DirectoryPrincipal, DirectorySearchInput } from "./directory/directory.js";
 export { DocumentCreateInput, DocumentDetail, DocumentExcerpt, DocumentGetInput, DocumentUpdateInput, } from "./documents/documents.js";
 export { ErrorCode, ErrorCurrent, ErrorIssue, ErrorResponse, errorCodes, invalidInput, Warning, } from "./error/error.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anchrd/colab-sdk",
-  "version": "0.1.19",
+  "version": "0.1.21",
   "description": "Types, Zod schemas and the service client of Colab, the contract between its UI, backends and outside services.",
   "type": "module",
   "license": "UNLICENSED",