@native-knowledge/knowledge-sdk 0.0.0-stage → 0.1.0-rc.2

package/dist/internal/contracts/tenant-web-crawl.d.ts

@@ -0,0 +1,310 @@
+/** 可执行静态网站任务；独立于冻结的 S1 意图，Tenant 是唯一归属。 */
+import { z } from 'zod';
+import { type TenantWebSpec } from './tenant-web-collection.js';
+export declare const TENANT_WEB_CRAWL_CAPABILITY = "TENANT_WEB_CRAWL_V1";
+export declare const tenantWebPolicySchema: z.ZodObject<{
+    ref: z.ZodString;
+    revision: z.ZodString;
+    acquisition: z.ZodEnum<{
+        ALLOW: "ALLOW";
+        DENY: "DENY";
+    }>;
+    propagation: z.ZodEnum<{
+        TENANT_ONLY: "TENANT_ONLY";
+        SHARE_ALLOWED: "SHARE_ALLOWED";
+        PUBLIC_ALLOWED: "PUBLIC_ALLOWED";
+    }>;
+}, z.core.$strict>;
+export declare const tenantWebCrawlSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<2>;
+    intent: z.ZodObject<{
+        schemaVersion: z.ZodLiteral<1>;
+        tenantId: z.ZodString;
+        requestId: z.ZodString;
+        jobRef: z.ZodString;
+        inputRevision: z.ZodString;
+        specRevision: z.ZodString;
+        requestedSpec: z.ZodObject<{
+            schemaVersion: z.ZodLiteral<1>;
+            url: z.ZodString;
+            scope: z.ZodObject<{
+                mode: z.ZodEnum<{
+                    FEED: "FEED";
+                    SINGLE_PAGE: "SINGLE_PAGE";
+                    URL_LIST: "URL_LIST";
+                    SITE_SECTION: "SITE_SECTION";
+                    SITEMAP: "SITEMAP";
+                }>;
+                allowedOrigins: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                includePaths: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                excludePaths: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                urls: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                followPagination: z.ZodDefault<z.ZodBoolean>;
+                followDetailLinks: z.ZodDefault<z.ZodBoolean>;
+            }, z.core.$strict>;
+            requirements: z.ZodObject<{
+                instruction: z.ZodString;
+                fields: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                    name: z.ZodString;
+                    type: z.ZodEnum<{
+                        string: "string";
+                        number: "number";
+                        boolean: "boolean";
+                        object: "object";
+                        array: "array";
+                    }>;
+                    required: z.ZodDefault<z.ZodBoolean>;
+                }, z.core.$strict>>>;
+                outputs: z.ZodDefault<z.ZodArray<z.ZodEnum<{
+                    RAW: "RAW";
+                    DOCUMENT: "DOCUMENT";
+                    RECORDS: "RECORDS";
+                }>>>;
+                attachments: z.ZodDefault<z.ZodArray<z.ZodEnum<{
+                    PDF: "PDF";
+                    IMAGE: "IMAGE";
+                    AUDIO: "AUDIO";
+                    VIDEO: "VIDEO";
+                    OTHER: "OTHER";
+                }>>>;
+            }, z.core.$strict>;
+            limits: z.ZodDefault<z.ZodObject<{
+                maxPages: z.ZodDefault<z.ZodNumber>;
+                maxDepth: z.ZodDefault<z.ZodNumber>;
+                maxResourceBytes: z.ZodDefault<z.ZodNumber>;
+                maxTotalBytes: z.ZodDefault<z.ZodNumber>;
+                maxRuntimeMs: z.ZodDefault<z.ZodNumber>;
+                maxInteractions: z.ZodDefault<z.ZodNumber>;
+                maxFiles: z.ZodDefault<z.ZodNumber>;
+            }, z.core.$strict>>;
+            refresh: z.ZodDefault<z.ZodObject<{
+                mode: z.ZodLiteral<"ONCE">;
+            }, z.core.$strict>>;
+        }, z.core.$strict>;
+        spec: z.ZodObject<{
+            schemaVersion: z.ZodLiteral<1>;
+            url: z.ZodString;
+            scope: z.ZodObject<{
+                mode: z.ZodEnum<{
+                    FEED: "FEED";
+                    SINGLE_PAGE: "SINGLE_PAGE";
+                    URL_LIST: "URL_LIST";
+                    SITE_SECTION: "SITE_SECTION";
+                    SITEMAP: "SITEMAP";
+                }>;
+                allowedOrigins: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                includePaths: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                excludePaths: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                urls: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                followPagination: z.ZodDefault<z.ZodBoolean>;
+                followDetailLinks: z.ZodDefault<z.ZodBoolean>;
+            }, z.core.$strict>;
+            requirements: z.ZodObject<{
+                instruction: z.ZodString;
+                fields: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                    name: z.ZodString;
+                    type: z.ZodEnum<{
+                        string: "string";
+                        number: "number";
+                        boolean: "boolean";
+                        object: "object";
+                        array: "array";
+                    }>;
+                    required: z.ZodDefault<z.ZodBoolean>;
+                }, z.core.$strict>>>;
+                outputs: z.ZodDefault<z.ZodArray<z.ZodEnum<{
+                    RAW: "RAW";
+                    DOCUMENT: "DOCUMENT";
+                    RECORDS: "RECORDS";
+                }>>>;
+                attachments: z.ZodDefault<z.ZodArray<z.ZodEnum<{
+                    PDF: "PDF";
+                    IMAGE: "IMAGE";
+                    AUDIO: "AUDIO";
+                    VIDEO: "VIDEO";
+                    OTHER: "OTHER";
+                }>>>;
+            }, z.core.$strict>;
+            limits: z.ZodDefault<z.ZodObject<{
+                maxPages: z.ZodDefault<z.ZodNumber>;
+                maxDepth: z.ZodDefault<z.ZodNumber>;
+                maxResourceBytes: z.ZodDefault<z.ZodNumber>;
+                maxTotalBytes: z.ZodDefault<z.ZodNumber>;
+                maxRuntimeMs: z.ZodDefault<z.ZodNumber>;
+                maxInteractions: z.ZodDefault<z.ZodNumber>;
+                maxFiles: z.ZodDefault<z.ZodNumber>;
+            }, z.core.$strict>>;
+            refresh: z.ZodDefault<z.ZodObject<{
+                mode: z.ZodLiteral<"ONCE">;
+            }, z.core.$strict>>;
+        }, z.core.$strict>;
+        actor: z.ZodObject<{
+            tenantId: z.ZodString;
+            userId: z.ZodString;
+            agentId: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        }, z.core.$strict>;
+        authority: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            type: z.ZodLiteral<"HOST">;
+        }, z.core.$strict>, z.ZodObject<{
+            type: z.ZodLiteral<"OAUTH">;
+            authorityVersion: z.ZodLiteral<2>;
+            clientId: z.ZodString;
+            grantId: z.ZodString;
+            resource: z.ZodString;
+        }, z.core.$strict>], "type">;
+        submittedAt: z.ZodISODateTime;
+    }, z.core.$strict>;
+    policy: z.ZodObject<{
+        ref: z.ZodString;
+        revision: z.ZodString;
+        acquisition: z.ZodEnum<{
+            ALLOW: "ALLOW";
+            DENY: "DENY";
+        }>;
+        propagation: z.ZodEnum<{
+            TENANT_ONLY: "TENANT_ONLY";
+            SHARE_ALLOWED: "SHARE_ALLOWED";
+            PUBLIC_ALLOWED: "PUBLIC_ALLOWED";
+        }>;
+    }, z.core.$strict>;
+}, z.core.$strict>;
+export type TenantWebCrawl = z.infer<typeof tenantWebCrawlSchema>;
+export declare function tenantWebAdmissionKey(tenantId: string, jobRef: string): string;
+export declare const tenantWebTargetSchema: z.ZodObject<{
+    ordinal: z.ZodNumber;
+    url: z.ZodString;
+    parent: z.ZodNullable<z.ZodString>;
+    depth: z.ZodNumber;
+    role: z.ZodEnum<{
+        FEED: "FEED";
+        SITEMAP: "SITEMAP";
+        PAGE: "PAGE";
+    }>;
+}, z.core.$strict>;
+export type TenantWebTarget = z.infer<typeof tenantWebTargetSchema>;
+export declare const tenantWebOutcomeSchema: z.ZodObject<{
+    ordinal: z.ZodNumber;
+    batchId: z.ZodNullable<z.ZodString>;
+    capture: z.ZodNullable<z.ZodObject<{
+        schemaVersion: z.ZodLiteral<1>;
+        tenantId: z.ZodString;
+        batchRef: z.ZodString;
+        sourceRevisionRef: z.ZodString;
+        taskId: z.ZodString;
+        evidenceRef: z.ZodString;
+        contentHash: z.ZodString;
+        byteLength: z.ZodNumber;
+        mediaType: z.ZodString;
+        observedAt: z.ZodISODateTime;
+        finalLocator: z.ZodString;
+        httpStatus: z.ZodNullable<z.ZodNumber>;
+        acquisitionRevision: z.ZodNullable<z.ZodString>;
+        headersRecorded: z.ZodBoolean;
+    }, z.core.$strict>>;
+    registered: z.ZodBoolean;
+    errorCode: z.ZodNullable<z.ZodString>;
+    reused: z.ZodBoolean;
+}, z.core.$strict>;
+export type TenantWebOutcome = z.infer<typeof tenantWebOutcomeSchema>;
+export declare const tenantWebHeadSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<1>;
+    seq: z.ZodNumber;
+    queued: z.ZodNumber;
+    cursor: z.ZodNumber;
+    stage: z.ZodEnum<{
+        ACQUIRE: "ACQUIRE";
+        DISCOVER: "DISCOVER";
+    }>;
+    after: z.ZodNumber;
+    attempts: z.ZodNumber;
+    currentAcquired: z.ZodBoolean;
+    acquired: z.ZodNumber;
+    registered: z.ZodNumber;
+    chargedBytes: z.ZodNumber;
+    chargedMs: z.ZodNumber;
+    reservation: z.ZodNullable<z.ZodObject<{
+        bytes: z.ZodNumber;
+        ms: z.ZodNumber;
+    }, z.core.$strict>>;
+    gaps: z.ZodArray<z.ZodString>;
+    state: z.ZodEnum<{
+        PARTIAL: "PARTIAL";
+        COMPLETE: "COMPLETE";
+        QUEUED: "QUEUED";
+        RUNNING: "RUNNING";
+    }>;
+    finishedAt: z.ZodNullable<z.ZodISODateTime>;
+    notBefore: z.ZodNullable<z.ZodISODateTime>;
+}, z.core.$strict>;
+export type TenantWebHead = z.infer<typeof tenantWebHeadSchema>;
+export declare const tenantWebChildOwnerSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<1>;
+    jobRef: z.ZodString;
+    ordinal: z.ZodNumber;
+    requestId: z.ZodString;
+    batchId: z.ZodString;
+}, z.core.$strict>;
+export type TenantWebChildOwner = z.infer<typeof tenantWebChildOwnerSchema>;
+export declare const tenantWebCrawlStatusSchema: z.ZodObject<{
+    requestId: z.ZodString;
+    jobRef: z.ZodString;
+    specRevision: z.ZodString;
+    state: z.ZodEnum<{
+        PARTIAL: "PARTIAL";
+        COMPLETE: "COMPLETE";
+        QUEUED: "QUEUED";
+        RUNNING: "RUNNING";
+    }>;
+    phase: z.ZodLiteral<"STATIC_ACQUISITION">;
+    requirementsState: z.ZodLiteral<"DECLARED_NOT_EVALUATED">;
+    indexState: z.ZodLiteral<"NOT_EVALUATED">;
+    coverage: z.ZodObject<{
+        total: z.ZodNullable<z.ZodNumber>;
+        discovered: z.ZodNumber;
+        acquired: z.ZodNumber;
+        registered: z.ZodNumber;
+        pending: z.ZodNumber;
+        gaps: z.ZodArray<z.ZodString>;
+    }, z.core.$strict>;
+    budget: z.ZodObject<{
+        chargedBytes: z.ZodNumber;
+        chargedMs: z.ZodNumber;
+        conservativeReservation: z.ZodBoolean;
+    }, z.core.$strict>;
+    finishedAt: z.ZodNullable<z.ZodISODateTime>;
+    nextEligibleAt: z.ZodNullable<z.ZodISODateTime>;
+}, z.core.$strict>;
+export declare function tenantWebCrawlStatus(job: TenantWebCrawl, head: TenantWebHead): {
+    requestId: string;
+    jobRef: string;
+    specRevision: string;
+    state: "PARTIAL" | "COMPLETE" | "QUEUED" | "RUNNING";
+    phase: "STATIC_ACQUISITION";
+    requirementsState: "DECLARED_NOT_EVALUATED";
+    indexState: "NOT_EVALUATED";
+    coverage: {
+        total: number | null;
+        discovered: number;
+        acquired: number;
+        registered: number;
+        pending: number;
+        gaps: string[];
+    };
+    budget: {
+        chargedBytes: number;
+        chargedMs: number;
+        conservativeReservation: boolean;
+    };
+    finishedAt: string | null;
+    nextEligibleAt: string | null;
+};
+export declare function tenantWebPathMatches(pattern: string, pathname: string): boolean;
+export declare function tenantWebInScope(spec: Pick<TenantWebSpec, 'url' | 'scope'>, raw: string, options?: {
+    robots?: boolean;
+    bootstrap?: boolean;
+}): boolean;
+export declare function sameTenantWebDelegation(a: Pick<TenantWebCrawl, 'policy'> & {
+    intent: Pick<TenantWebCrawl['intent'], 'tenantId' | 'authority' | 'actor'>;
+}, b: Pick<TenantWebCrawl, 'policy'> & {
+    intent: Pick<TenantWebCrawl['intent'], 'tenantId' | 'authority' | 'actor'>;
+}): boolean;

package/dist/internal/contracts/tenant-web-crawl.js

@@ -0,0 +1,55 @@
+/** 可执行静态网站任务；独立于冻结的 S1 意图，Tenant 是唯一归属。 */
+import { z } from 'zod';
+import { createHash } from 'node:crypto';
+import { tenantStorageKey } from './resource-scope.js';
+import { tenantWebIntentSchema, tenantWebUrlSchema, tenantWebJobRefSchema, tenantWebCanonical } from './tenant-web-collection.js';
+import { tenantSourceSemanticsSchema } from './tenant-source.js';
+import { tenantCollectionCaptureSchema, tenantCollectionCycleSchema } from './tenant-collection.js';
+import { isoDateTimeSchema, sha256Schema } from './common.js';
+export const TENANT_WEB_CRAWL_CAPABILITY = 'TENANT_WEB_CRAWL_V1';
+export const tenantWebPolicySchema = tenantSourceSemanticsSchema.shape.sourcePolicy.refine(v => v.acquisition === 'ALLOW' && v.propagation === 'TENANT_ONLY');
+export const tenantWebCrawlSchema = z.object({ schemaVersion: z.literal(2), intent: tenantWebIntentSchema, policy: tenantWebPolicySchema }).strict();
+export function tenantWebAdmissionKey(tenantId, jobRef) { tenantWebJobRefSchema.parse(jobRef); if (new URL(jobRef).hostname !== tenantStorageKey(tenantId))
+    throw new Error('TENANT_WEB_SCOPE_FORBIDDEN'); return createHash('sha256').update(JSON.stringify(['TENANT_WEB_ADMISSION_V1', tenantId, jobRef])).digest('hex'); }
+export const tenantWebTargetSchema = z.object({ ordinal: z.number().int().min(0).max(9999), url: tenantWebUrlSchema, parent: tenantWebUrlSchema.nullable(), depth: z.number().int().min(0).max(10), role: z.enum(['PAGE', 'SITEMAP', 'FEED']) }).strict();
+export const tenantWebOutcomeSchema = z.object({ ordinal: tenantWebTargetSchema.shape.ordinal, batchId: z.string().regex(/^batch_[a-f0-9]{64}$/).nullable(), capture: tenantCollectionCaptureSchema.nullable(), registered: z.boolean(), errorCode: z.string().regex(/^[A-Z][A-Z0-9_]{0,127}$/).nullable(), reused: z.boolean() }).strict().refine(v => !v.registered || v.capture !== null);
+export const tenantWebHeadSchema = z.object({ schemaVersion: z.literal(1), seq: z.number().int().nonnegative(), queued: z.number().int().min(0).max(10000), cursor: z.number().int().min(0).max(10000), stage: z.enum(['ACQUIRE', 'DISCOVER']), after: z.number().int().nonnegative(), attempts: z.number().int().min(0).max(3), currentAcquired: z.boolean(), acquired: z.number().int().min(0).max(10000), registered: z.number().int().min(0).max(10000), chargedBytes: z.number().int().nonnegative(), chargedMs: z.number().int().nonnegative(), reservation: z.object({ bytes: z.number().int().nonnegative(), ms: z.number().int().nonnegative() }).strict().nullable(), gaps: z.array(z.string().regex(/^[A-Z][A-Z0-9_]{0,127}$/)).max(32), state: z.enum(['QUEUED', 'RUNNING', 'COMPLETE', 'PARTIAL']), finishedAt: isoDateTimeSchema.nullable(), notBefore: isoDateTimeSchema.nullable() }).strict().refine(v => v.cursor <= v.queued && v.registered <= v.acquired && v.acquired <= v.queued && (['COMPLETE', 'PARTIAL'].includes(v.state) === (v.finishedAt !== null)) && (!v.reservation || v.state === 'RUNNING' && v.chargedBytes >= v.reservation.bytes && v.chargedMs >= v.reservation.ms) && (!v.notBefore || v.state === 'RUNNING' && !v.reservation) && (v.state !== 'PARTIAL' || v.gaps.length > 0) && (v.state !== 'COMPLETE' || v.queued > 0 && v.cursor === v.queued && v.gaps.length === 0 && v.registered === v.queued && !v.reservation && !v.notBefore));
+export const tenantWebChildOwnerSchema = z.object({ schemaVersion: z.literal(1), jobRef: tenantWebJobRefSchema, ordinal: tenantWebTargetSchema.shape.ordinal, requestId: tenantCollectionCycleSchema, batchId: z.string().regex(/^batch_[a-f0-9]{64}$/) }).strict();
+export const tenantWebCrawlStatusSchema = z.object({ requestId: tenantCollectionCycleSchema, jobRef: tenantWebJobRefSchema, specRevision: sha256Schema, state: tenantWebHeadSchema.shape.state, phase: z.literal('STATIC_ACQUISITION'), requirementsState: z.literal('DECLARED_NOT_EVALUATED'), indexState: z.literal('NOT_EVALUATED'), coverage: z.object({ total: z.number().int().nonnegative().nullable(), discovered: z.number().int().nonnegative(), acquired: z.number().int().nonnegative(), registered: z.number().int().nonnegative(), pending: z.number().int().nonnegative(), gaps: tenantWebHeadSchema.shape.gaps }).strict(), budget: z.object({ chargedBytes: z.number().int().nonnegative(), chargedMs: z.number().int().nonnegative(), conservativeReservation: z.boolean() }).strict(), finishedAt: isoDateTimeSchema.nullable(), nextEligibleAt: isoDateTimeSchema.nullable() }).strict();
+export function tenantWebCrawlStatus(job, head) { return tenantWebCrawlStatusSchema.parse({ requestId: job.intent.requestId, jobRef: job.intent.jobRef, specRevision: job.intent.specRevision, state: head.state, phase: 'STATIC_ACQUISITION', requirementsState: 'DECLARED_NOT_EVALUATED', indexState: 'NOT_EVALUATED', coverage: { total: head.state === 'COMPLETE' ? head.queued : null, discovered: head.queued, acquired: head.acquired, registered: head.registered, pending: head.queued - head.cursor, gaps: head.gaps }, budget: { chargedBytes: head.chargedBytes, chargedMs: head.chargedMs, conservativeReservation: head.reservation !== null }, finishedAt: head.finishedAt, nextEligibleAt: head.notBefore }); }
+export function tenantWebPathMatches(pattern, pathname) { return pattern.endsWith('/**') ? (pathname === pattern.slice(0, -3) || pathname.startsWith(pattern.slice(0, -2))) : pathname === pattern; }
+export function tenantWebInScope(spec, raw, options = {}) {
+    const parsed = tenantWebUrlSchema.safeParse(raw);
+    if (!parsed.success)
+        return false;
+    const url = new URL(parsed.data), scope = spec.scope;
+    if (!scope.allowedOrigins.includes(url.origin))
+        return false;
+    if (options.robots)
+        return url.pathname === '/robots.txt' && !url.search;
+    let pathname;
+    try {
+        pathname = decodeURIComponent(url.pathname);
+    }
+    catch {
+        return false;
+    }
+    if (/[\\\x00-\x1f]/.test(pathname) || /%[0-9a-f]{2}/i.test(pathname))
+        return false;
+    // GitLab and similar public APIs encode a slash inside an identifier. Only an exact
+    // caller-listed URL without path filters can use it; section/prefix matching stays closed.
+    if (/%2f/i.test(url.pathname) && (!['SINGLE_PAGE', 'URL_LIST'].includes(scope.mode) || scope.includePaths.length > 0 || scope.excludePaths.length > 0))
+        return false;
+    if (scope.excludePaths.some(p => tenantWebPathMatches(p, pathname)))
+        return false;
+    if (options.bootstrap && raw === spec.url && ['SITEMAP', 'FEED'].includes(scope.mode))
+        return true;
+    if (scope.includePaths.length && !scope.includePaths.some(p => tenantWebPathMatches(p, pathname)))
+        return false;
+    if (scope.mode === 'SINGLE_PAGE')
+        return raw === spec.url;
+    if (scope.mode === 'URL_LIST')
+        return raw === spec.url || scope.urls.includes(raw);
+    return true;
+}
+export function sameTenantWebDelegation(a, b) { return a.intent.tenantId === b.intent.tenantId && tenantWebCanonical(a.policy) === tenantWebCanonical(b.policy) && tenantWebCanonical(a.intent.authority) === tenantWebCanonical(b.intent.authority) && tenantWebCanonical(a.intent.actor) === tenantWebCanonical(b.intent.actor); }