@x12i/content-fetcher 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +133 -0
- package/dist/adapters/default-http-client.d.ts +17 -0
- package/dist/adapters/default-http-client.d.ts.map +1 -0
- package/dist/adapters/default-http-client.js +74 -0
- package/dist/adapters/default-http-client.js.map +1 -0
- package/dist/channels.d.ts +34 -0
- package/dist/channels.d.ts.map +1 -0
- package/dist/channels.js +164 -0
- package/dist/channels.js.map +1 -0
- package/dist/classify.d.ts +13 -0
- package/dist/classify.d.ts.map +1 -0
- package/dist/classify.js +122 -0
- package/dist/classify.js.map +1 -0
- package/dist/config.d.ts +7 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +197 -0
- package/dist/config.js.map +1 -0
- package/dist/create-content-fetcher.d.ts +9 -0
- package/dist/create-content-fetcher.d.ts.map +1 -0
- package/dist/create-content-fetcher.js +836 -0
- package/dist/create-content-fetcher.js.map +1 -0
- package/dist/errors.d.ts +42 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +73 -0
- package/dist/errors.js.map +1 -0
- package/dist/handoff.d.ts +8 -0
- package/dist/handoff.d.ts.map +1 -0
- package/dist/handoff.js +20 -0
- package/dist/handoff.js.map +1 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +14 -0
- package/dist/index.js.map +1 -0
- package/dist/ratelimit.d.ts +27 -0
- package/dist/ratelimit.d.ts.map +1 -0
- package/dist/ratelimit.js +43 -0
- package/dist/ratelimit.js.map +1 -0
- package/dist/robots/robots-service.d.ts +40 -0
- package/dist/robots/robots-service.d.ts.map +1 -0
- package/dist/robots/robots-service.js +131 -0
- package/dist/robots/robots-service.js.map +1 -0
- package/dist/robots/robots-txt.d.ts +24 -0
- package/dist/robots/robots-txt.d.ts.map +1 -0
- package/dist/robots/robots-txt.js +91 -0
- package/dist/robots/robots-txt.js.map +1 -0
- package/dist/runtime-utils.d.ts +11 -0
- package/dist/runtime-utils.d.ts.map +1 -0
- package/dist/runtime-utils.js +32 -0
- package/dist/runtime-utils.js.map +1 -0
- package/dist/security/dns.d.ts +33 -0
- package/dist/security/dns.d.ts.map +1 -0
- package/dist/security/dns.js +60 -0
- package/dist/security/dns.js.map +1 -0
- package/dist/security/headers.d.ts +16 -0
- package/dist/security/headers.d.ts.map +1 -0
- package/dist/security/headers.js +54 -0
- package/dist/security/headers.js.map +1 -0
- package/dist/security/net-guards.d.ts +9 -0
- package/dist/security/net-guards.d.ts.map +1 -0
- package/dist/security/net-guards.js +102 -0
- package/dist/security/net-guards.js.map +1 -0
- package/dist/security/url.d.ts +32 -0
- package/dist/security/url.d.ts.map +1 -0
- package/dist/security/url.js +82 -0
- package/dist/security/url.js.map +1 -0
- package/dist/storage.d.ts +38 -0
- package/dist/storage.d.ts.map +1 -0
- package/dist/storage.js +109 -0
- package/dist/storage.js.map +1 -0
- package/dist/types.d.ts +232 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +9 -0
- package/dist/types.js.map +1 -0
- package/dist/util/bytes.d.ts +21 -0
- package/dist/util/bytes.d.ts.map +1 -0
- package/dist/util/bytes.js +44 -0
- package/dist/util/bytes.js.map +1 -0
- package/dist/util/hash.d.ts +5 -0
- package/dist/util/hash.d.ts.map +1 -0
- package/dist/util/hash.js +11 -0
- package/dist/util/hash.js.map +1 -0
- package/dist/util/ids.d.ts +3 -0
- package/dist/util/ids.d.ts.map +1 -0
- package/dist/util/ids.js +8 -0
- package/dist/util/ids.js.map +1 -0
- package/dist/validate-request.d.ts +9 -0
- package/dist/validate-request.d.ts.map +1 -0
- package/dist/validate-request.js +137 -0
- package/dist/validate-request.js.map +1 -0
- package/package.json +34 -0
package/dist/storage.js
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { ArtifactStoreError } from "./errors.js";
|
|
2
|
+
import { sha256ContentHash } from "./util/hash.js";
|
|
3
|
+
async function storeOperation(operation, run) {
|
|
4
|
+
try {
|
|
5
|
+
return await run();
|
|
6
|
+
}
|
|
7
|
+
catch (error) {
|
|
8
|
+
if (error instanceof ArtifactStoreError)
|
|
9
|
+
throw error;
|
|
10
|
+
throw new ArtifactStoreError(`Artifact store operation '${operation}' failed.`, operation);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Rights-gated, deduplicating persistence of raw artifacts:
|
|
15
|
+
* - raw bytes are stored only when the rights profile allows retention;
|
|
16
|
+
* - content already known for this source is never re-stored or overwritten;
|
|
17
|
+
* - every acquisition keeps its own record via putMetadata.
|
|
18
|
+
*/
|
|
19
|
+
export class ArtifactPersistence {
|
|
20
|
+
store;
|
|
21
|
+
constructor(store) {
|
|
22
|
+
this.store = store;
|
|
23
|
+
}
|
|
24
|
+
async persistArtifact(input) {
|
|
25
|
+
const contentHash = sha256ContentHash(input.bytes);
|
|
26
|
+
if (!input.rights.allowRawRetention) {
|
|
27
|
+
return {};
|
|
28
|
+
}
|
|
29
|
+
const existing = await storeOperation("findByContentHash", () => this.store.findByContentHash({ sourceId: input.sourceId, contentHash }));
|
|
30
|
+
if (existing !== null) {
|
|
31
|
+
return {
|
|
32
|
+
artifact: {
|
|
33
|
+
artifactId: existing.artifactId,
|
|
34
|
+
fileName: input.fileName,
|
|
35
|
+
contentType: input.contentType,
|
|
36
|
+
contentHash,
|
|
37
|
+
byteLength: input.bytes.length,
|
|
38
|
+
},
|
|
39
|
+
duplicateOfArtifactId: existing.artifactId,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
const stored = await storeOperation("putOriginal", () => this.store.putOriginal({
|
|
43
|
+
acquisitionId: input.acquisitionId,
|
|
44
|
+
fileName: input.fileName,
|
|
45
|
+
contentType: input.contentType,
|
|
46
|
+
bytes: input.bytes,
|
|
47
|
+
contentHash,
|
|
48
|
+
rights: input.rights,
|
|
49
|
+
}));
|
|
50
|
+
return {
|
|
51
|
+
artifact: {
|
|
52
|
+
artifactId: stored.artifactId,
|
|
53
|
+
fileName: input.fileName,
|
|
54
|
+
contentType: input.contentType,
|
|
55
|
+
contentHash,
|
|
56
|
+
byteLength: input.bytes.length,
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
/** Resolve a previous version's artifactId when the caller supplied its content hash. */
|
|
61
|
+
async resolvePreviousVersionId(request, policy) {
|
|
62
|
+
const hash = request.previousVersion?.contentHash;
|
|
63
|
+
if (hash === undefined)
|
|
64
|
+
return undefined;
|
|
65
|
+
const found = await storeOperation("findByContentHash", () => this.store.findByContentHash({ sourceId: policy.sourceId, contentHash: hash }));
|
|
66
|
+
return found?.artifactId;
|
|
67
|
+
}
|
|
68
|
+
/** Metadata + rights decision for one acquisition event. Never contains raw content or credentials. */
|
|
69
|
+
async writeAcquisitionMetadata(result, extras) {
|
|
70
|
+
await storeOperation("putMetadata", () => this.store.putMetadata({
|
|
71
|
+
acquisitionId: result.acquisitionId,
|
|
72
|
+
metadata: {
|
|
73
|
+
recordType: "acquisition",
|
|
74
|
+
...extras,
|
|
75
|
+
acquisitionId: result.acquisitionId,
|
|
76
|
+
requestId: result.requestId,
|
|
77
|
+
sourceId: result.sourceId,
|
|
78
|
+
status: result.status,
|
|
79
|
+
channel: result.channel,
|
|
80
|
+
artifact: result.artifact,
|
|
81
|
+
version: result.version,
|
|
82
|
+
provenance: result.provenance,
|
|
83
|
+
rights: result.rights,
|
|
84
|
+
diagnostics: result.diagnostics,
|
|
85
|
+
},
|
|
86
|
+
}));
|
|
87
|
+
}
|
|
88
|
+
/** Compact audit record for every completed request. Sanitized by construction. */
|
|
89
|
+
async writeAuditRecord(result, requestedAt, completedAt) {
|
|
90
|
+
await storeOperation("putMetadata", () => this.store.putMetadata({
|
|
91
|
+
acquisitionId: result.acquisitionId,
|
|
92
|
+
metadata: {
|
|
93
|
+
recordType: "audit",
|
|
94
|
+
requestId: result.requestId,
|
|
95
|
+
acquisitionId: result.acquisitionId,
|
|
96
|
+
sourceId: result.sourceId,
|
|
97
|
+
status: result.status,
|
|
98
|
+
channel: result.channel,
|
|
99
|
+
requestedAt: requestedAt.toISOString(),
|
|
100
|
+
completedAt: completedAt.toISOString(),
|
|
101
|
+
...(result.artifact?.contentHash !== undefined
|
|
102
|
+
? { contentHash: result.artifact.contentHash }
|
|
103
|
+
: {}),
|
|
104
|
+
rightsProfileId: result.rights.profileId,
|
|
105
|
+
},
|
|
106
|
+
}));
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=storage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage.js","sourceRoot":"","sources":["../src/storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AA+BnD,KAAK,UAAU,cAAc,CAAI,SAAiB,EAAE,GAAqB;IACvE,IAAI,CAAC;QACH,OAAO,MAAM,GAAG,EAAE,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,kBAAkB;YAAE,MAAM,KAAK,CAAC;QACrD,MAAM,IAAI,kBAAkB,CAC1B,6BAA6B,SAAS,WAAW,EACjD,SAAS,CACV,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,OAAO,mBAAmB;IACD;IAA7B,YAA6B,KAAoB;QAApB,UAAK,GAAL,KAAK,CAAe;IAAG,CAAC;IAErD,KAAK,CAAC,eAAe,CAAC,KAA2B;QAC/C,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACpC,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,mBAAmB,EAAE,GAAG,EAAE,CAC9D,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,CACxE,CAAC;QACF,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YACtB,OAAO;gBACL,QAAQ,EAAE;oBACR,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,WAAW;oBACX,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM;iBAC/B;gBACD,qBAAqB,EAAE,QAAQ,CAAC,UAAU;aAC3C,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CACtD,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;YACrB,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,WAAW;YACX,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC,CACH,CAAC;QACF,OAAO;YACL,QAAQ,EAAE;gBACR,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,WAAW;gBACX,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM;aAC/B;SACF,CAAC;IACJ,CAAC;IAED,yFAAyF;IACzF,KAAK,CAAC,wBAAwB,CAC5B,OAA8C,EAC9C,MAAsC;QAEtC,MAAM,IAAI,GAAG,OAAO,CAAC,eAAe,EAAE,WAAW,CAAC;QAClD,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QACzC,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,mBAAmB,EAAE,GAAG,EAAE,CAC3D,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAC/E,CAAC;QACF,OAAO,KAAK,EAAE,UAAU,CAAC;IAC3B,CAAC;IAED,uGAAuG;IACvG,KAAK,CAAC,wBAAwB,CAAC,MAAyB,EAAE,MAAc;QACtE,MAAM,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CACvC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;YACrB,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,QAAQ,EAAE;gBACR,UAAU,EAAE,aAAa;gBACzB,GAAG,MAAM;gBACT,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,WAAW,EAAE,MAAM,CAAC,WAAW;aAChC;SACF,CAAC,CACH,CAAC;IACJ,CAAC;IAED,mFAAmF;IACnF,KAAK,CAAC,gBAAgB,CACpB,MAAyB,EACzB,WAAiB,EACjB,WAAiB;QAEjB,MAAM,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CACvC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;YACrB,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,QAAQ,EAAE;gBACR,UAAU,EAAE,OAAO;gBACnB,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE;gBACtC,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE;gBACtC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,WAAW,KAAK,SAAS;oBAC5C,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE;oBAC9C,CAAC,CAAC,EAAE,CAAC;gBACP,eAAe,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS;aACzC;SACF,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public types for @x12i/content-fetcher.
|
|
3
|
+
*
|
|
4
|
+
* These shapes are part of the package contract and mirror the package
|
|
5
|
+
* requirement document exactly. Downstream packages (extractor, normalizer)
|
|
6
|
+
* consume {@link FetchedArtifactReference}; do not rename or widen fields.
|
|
7
|
+
*/
|
|
8
|
+
/** Content hash format used throughout the package. */
|
|
9
|
+
export type Sha256ContentHash = `sha256:${string}`;
|
|
10
|
+
/** Ways content may be acquired from an approved source. */
|
|
11
|
+
export type AcquisitionChannel = "direct_http" | "official_api" | "rss_or_atom" | "permitted_browser" | "approved_archive" | "licensed_provider" | "manual_file";
|
|
12
|
+
/** Terminal outcome of an acquisition request. There is no generic "failed". */
|
|
13
|
+
export type AcquisitionStatus = "collected" | "not_modified" | "metadata_only" | "robots_disallowed" | "rate_limited" | "waf_or_bot_challenge" | "login_required" | "paywalled" | "geo_restricted" | "source_not_authorised" | "not_found" | "temporarily_unavailable" | "redirect_disallowed" | "content_too_large" | "unsupported_content_type" | "integrity_error" | "manual_review_required";
|
|
14
|
+
/** Rights profile attached to every result and stored with every artifact. */
|
|
15
|
+
export interface RightsProfile {
|
|
16
|
+
profileId: string;
|
|
17
|
+
allowRawRetention: boolean;
|
|
18
|
+
allowExtraction: boolean;
|
|
19
|
+
allowDisplay: boolean;
|
|
20
|
+
allowTraining: boolean;
|
|
21
|
+
retentionClass: string;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Policy controlling every acquisition for one source. A request is only
|
|
25
|
+
* executed when it satisfies the source's policy.
|
|
26
|
+
*/
|
|
27
|
+
export interface SourcePolicy {
|
|
28
|
+
sourceId: string;
|
|
29
|
+
publisher: string;
|
|
30
|
+
/** Hostnames permitted for this source. A hostname or any subdomain of an entry matches. */
|
|
31
|
+
allowedDomains: string[];
|
|
32
|
+
allowedProtocols: Array<"https" | "http">;
|
|
33
|
+
allowedChannels: AcquisitionChannel[];
|
|
34
|
+
/** Channel fallback order. Must be a subset of {@link allowedChannels}. */
|
|
35
|
+
channelOrder: AcquisitionChannel[];
|
|
36
|
+
robotsPolicy: "required" | "not_applicable";
|
|
37
|
+
browserAccess: "disabled" | "permitted";
|
|
38
|
+
authProfileId?: string;
|
|
39
|
+
archiveAccess: "disabled" | "permitted";
|
|
40
|
+
licensedProviderAccess: "disabled" | "permitted";
|
|
41
|
+
rateLimit: {
|
|
42
|
+
maxRequestsPerMinute: number;
|
|
43
|
+
maxConcurrentRequests: number;
|
|
44
|
+
};
|
|
45
|
+
limits: {
|
|
46
|
+
maxRedirects: number;
|
|
47
|
+
maxResponseBytes: number;
|
|
48
|
+
};
|
|
49
|
+
rights: RightsProfile;
|
|
50
|
+
}
|
|
51
|
+
/** A file supplied directly by an authorized user or system. */
|
|
52
|
+
export interface ManualFileInput {
|
|
53
|
+
fileName: string;
|
|
54
|
+
contentType: string;
|
|
55
|
+
bytes: Uint8Array;
|
|
56
|
+
}
|
|
57
|
+
/** Exactly one field must be supplied. */
|
|
58
|
+
export interface FetchRequestResource {
|
|
59
|
+
url?: string;
|
|
60
|
+
apiResource?: string;
|
|
61
|
+
feedEntryUrl?: string;
|
|
62
|
+
archiveReference?: string;
|
|
63
|
+
licensedResourceId?: string;
|
|
64
|
+
manualFile?: ManualFileInput;
|
|
65
|
+
}
|
|
66
|
+
/** Version identifiers from a previous acquisition of the same resource. */
|
|
67
|
+
export interface PreviousVersion {
|
|
68
|
+
etag?: string;
|
|
69
|
+
lastModified?: string;
|
|
70
|
+
contentHash?: Sha256ContentHash;
|
|
71
|
+
}
|
|
72
|
+
/** The exact resource to acquire. The fetcher never locates related URLs. */
|
|
73
|
+
export interface FetchRequest {
|
|
74
|
+
requestId: string;
|
|
75
|
+
sourceId: string;
|
|
76
|
+
resource: FetchRequestResource;
|
|
77
|
+
expectedContentTypes?: string[];
|
|
78
|
+
previousVersion?: PreviousVersion;
|
|
79
|
+
}
|
|
80
|
+
/** Public interface returned by {@link createContentFetcher}. */
|
|
81
|
+
export interface ContentFetcher {
|
|
82
|
+
fetch(request: FetchRequest): Promise<AcquisitionResult>;
|
|
83
|
+
}
|
|
84
|
+
export interface AcquisitionArtifact {
|
|
85
|
+
artifactId: string;
|
|
86
|
+
fileName: string;
|
|
87
|
+
contentType: string;
|
|
88
|
+
contentHash: Sha256ContentHash;
|
|
89
|
+
byteLength: number;
|
|
90
|
+
}
|
|
91
|
+
export interface AcquisitionVersion {
|
|
92
|
+
etag?: string;
|
|
93
|
+
lastModified?: string;
|
|
94
|
+
previousVersionId?: string;
|
|
95
|
+
duplicateOfArtifactId?: string;
|
|
96
|
+
}
|
|
97
|
+
export interface AcquisitionProvenance {
|
|
98
|
+
requestedUrl?: string;
|
|
99
|
+
finalUrl?: string;
|
|
100
|
+
retrievedAt: string;
|
|
101
|
+
publisher: string;
|
|
102
|
+
archiveReference?: string;
|
|
103
|
+
feedEntryId?: string;
|
|
104
|
+
apiResource?: string;
|
|
105
|
+
}
|
|
106
|
+
export interface AcquisitionDiagnostics {
|
|
107
|
+
httpStatus?: number;
|
|
108
|
+
retryAfterSeconds?: number;
|
|
109
|
+
reasonCode?: string;
|
|
110
|
+
sanitizedMessage?: string;
|
|
111
|
+
}
|
|
112
|
+
/** Returned for every request that passes input validation. Never thrown for expected retrieval outcomes. */
|
|
113
|
+
export interface AcquisitionResult {
|
|
114
|
+
requestId: string;
|
|
115
|
+
acquisitionId: string;
|
|
116
|
+
status: AcquisitionStatus;
|
|
117
|
+
sourceId: string;
|
|
118
|
+
channel?: AcquisitionChannel;
|
|
119
|
+
artifact?: AcquisitionArtifact;
|
|
120
|
+
version?: AcquisitionVersion;
|
|
121
|
+
provenance: AcquisitionProvenance;
|
|
122
|
+
rights: RightsProfile;
|
|
123
|
+
diagnostics: AcquisitionDiagnostics;
|
|
124
|
+
}
|
|
125
|
+
/** Handoff payload for the downstream extraction / normalization package. */
|
|
126
|
+
export interface FetchedArtifactReference {
|
|
127
|
+
acquisitionId: string;
|
|
128
|
+
artifactId: string;
|
|
129
|
+
sourceId: string;
|
|
130
|
+
fileName: string;
|
|
131
|
+
contentType: string;
|
|
132
|
+
contentHash: Sha256ContentHash;
|
|
133
|
+
provenance: AcquisitionProvenance;
|
|
134
|
+
rights: RightsProfile;
|
|
135
|
+
}
|
|
136
|
+
/** Immutable raw-artifact storage owned by the host application. */
|
|
137
|
+
export interface ArtifactStore {
|
|
138
|
+
putOriginal(input: {
|
|
139
|
+
acquisitionId: string;
|
|
140
|
+
fileName: string;
|
|
141
|
+
contentType: string;
|
|
142
|
+
bytes: Uint8Array;
|
|
143
|
+
contentHash: Sha256ContentHash;
|
|
144
|
+
rights: RightsProfile;
|
|
145
|
+
}): Promise<{
|
|
146
|
+
artifactId: string;
|
|
147
|
+
}>;
|
|
148
|
+
putMetadata(input: {
|
|
149
|
+
acquisitionId: string;
|
|
150
|
+
metadata: object;
|
|
151
|
+
}): Promise<void>;
|
|
152
|
+
findByContentHash(input: {
|
|
153
|
+
sourceId: string;
|
|
154
|
+
contentHash: Sha256ContentHash;
|
|
155
|
+
}): Promise<{
|
|
156
|
+
artifactId: string;
|
|
157
|
+
} | null>;
|
|
158
|
+
}
|
|
159
|
+
/** Minimal HTTP transport. Implementations must NOT follow redirects automatically. */
|
|
160
|
+
export interface HttpClientRequestInit {
|
|
161
|
+
method?: "GET" | "HEAD";
|
|
162
|
+
headers?: Record<string, string>;
|
|
163
|
+
signal?: AbortSignal;
|
|
164
|
+
}
|
|
165
|
+
export interface HttpClientResponse {
|
|
166
|
+
status: number;
|
|
167
|
+
/** Header names in lowercase. */
|
|
168
|
+
headers: Record<string, string>;
|
|
169
|
+
body?: AsyncIterable<Uint8Array>;
|
|
170
|
+
}
|
|
171
|
+
export interface HttpClient {
|
|
172
|
+
request(url: string, init?: HttpClientRequestInit): Promise<HttpClientResponse>;
|
|
173
|
+
}
|
|
174
|
+
/** Result of a browser adapter retrieval. The adapter must not attempt to bypass challenges. */
|
|
175
|
+
export interface BrowserFetchOutput {
|
|
176
|
+
/** HTTP-like status the browser interaction produced. */
|
|
177
|
+
status: number;
|
|
178
|
+
finalUrl?: string;
|
|
179
|
+
contentType?: string;
|
|
180
|
+
bytes?: Uint8Array;
|
|
181
|
+
headers?: Record<string, string>;
|
|
182
|
+
/** Set when the page was not retrievable for an access-restriction reason. */
|
|
183
|
+
blockedReason?: "captcha" | "waf" | "login" | "paywall" | "geo";
|
|
184
|
+
}
|
|
185
|
+
export interface BrowserAdapter {
|
|
186
|
+
fetchPage(input: {
|
|
187
|
+
url: string;
|
|
188
|
+
userAgent: string;
|
|
189
|
+
}): Promise<BrowserFetchOutput>;
|
|
190
|
+
}
|
|
191
|
+
/** Credentials supplied for a policy's authProfileId. Values are never persisted or logged. */
|
|
192
|
+
export interface AuthCredentials {
|
|
193
|
+
type: "bearer" | "header";
|
|
194
|
+
/** Header name for {@link type}:"header". Defaults to x-api-key. */
|
|
195
|
+
headerName?: string;
|
|
196
|
+
value: string;
|
|
197
|
+
}
|
|
198
|
+
export interface AuthProvider {
|
|
199
|
+
getCredentials(input: {
|
|
200
|
+
authProfileId: string;
|
|
201
|
+
url: string;
|
|
202
|
+
channel: AcquisitionChannel;
|
|
203
|
+
}): Promise<AuthCredentials | null>;
|
|
204
|
+
}
|
|
205
|
+
/** Address resolution hook so SSRF validation can run after DNS resolution. */
|
|
206
|
+
export interface DnsResolver {
|
|
207
|
+
lookup(hostname: string): Promise<{
|
|
208
|
+
addresses: string[];
|
|
209
|
+
}>;
|
|
210
|
+
}
|
|
211
|
+
/** Options for {@link createContentFetcher}. */
|
|
212
|
+
export interface ContentFetcherOptions {
|
|
213
|
+
sourcePolicies: SourcePolicy[];
|
|
214
|
+
artifactStore: ArtifactStore;
|
|
215
|
+
/** Required (at creation time) when any policy enables a network channel. */
|
|
216
|
+
httpClient?: HttpClient;
|
|
217
|
+
/** Required (at fetch time) when a permitted_browser acquisition is attempted. */
|
|
218
|
+
browserAdapter?: BrowserAdapter;
|
|
219
|
+
/** Required (at fetch time) when a policy declaring an authProfileId is used. */
|
|
220
|
+
authProvider?: AuthProvider;
|
|
221
|
+
/** Default: system resolver. */
|
|
222
|
+
dnsResolver?: DnsResolver;
|
|
223
|
+
/** User agent for requests and robots.txt evaluation. Default: "x12i-content-fetcher/1.0". */
|
|
224
|
+
userAgent?: string;
|
|
225
|
+
/** Default: 10 minutes. */
|
|
226
|
+
robotsCacheTtlMs?: number;
|
|
227
|
+
/** Default: 30_000 ms. */
|
|
228
|
+
requestTimeoutMs?: number;
|
|
229
|
+
/** Clock injection for tests. Default: () => new Date(). */
|
|
230
|
+
now?: () => Date;
|
|
231
|
+
}
|
|
232
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,uDAAuD;AACvD,MAAM,MAAM,iBAAiB,GAAG,UAAU,MAAM,EAAE,CAAC;AAEnD,4DAA4D;AAC5D,MAAM,MAAM,kBAAkB,GAC1B,aAAa,GACb,cAAc,GACd,aAAa,GACb,mBAAmB,GACnB,kBAAkB,GAClB,mBAAmB,GACnB,aAAa,CAAC;AAElB,gFAAgF;AAChF,MAAM,MAAM,iBAAiB,GACzB,WAAW,GACX,cAAc,GACd,eAAe,GACf,mBAAmB,GACnB,cAAc,GACd,sBAAsB,GACtB,gBAAgB,GAChB,WAAW,GACX,gBAAgB,GAChB,uBAAuB,GACvB,WAAW,GACX,yBAAyB,GACzB,qBAAqB,GACrB,mBAAmB,GACnB,0BAA0B,GAC1B,iBAAiB,GACjB,wBAAwB,CAAC;AAE7B,8EAA8E;AAC9E,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,eAAe,EAAE,OAAO,CAAC;IACzB,YAAY,EAAE,OAAO,CAAC;IACtB,aAAa,EAAE,OAAO,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAElB,4FAA4F;IAC5F,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,gBAAgB,EAAE,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC;IAE1C,eAAe,EAAE,kBAAkB,EAAE,CAAC;IACtC,2EAA2E;IAC3E,YAAY,EAAE,kBAAkB,EAAE,CAAC;IAEnC,YAAY,EAAE,UAAU,GAAG,gBAAgB,CAAC;IAC5C,aAAa,EAAE,UAAU,GAAG,WAAW,CAAC;IACxC,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,aAAa,EAAE,UAAU,GAAG,WAAW,CAAC;IACxC,sBAAsB,EAAE,UAAU,GAAG,WAAW,CAAC;IAEjD,SAAS,EAAE;QACT,oBAAoB,EAAE,MAAM,CAAC;QAC7B,qBAAqB,EAAE,MAAM,CAAC;KAC/B,CAAC;IAEF,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;KAC1B,CAAC;IAEF,MAAM,EAAE,aAAa,CAAC;CACvB;AAED,gEAAgE;AAChE,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,UAAU,CAAC;CACnB;AAED,0CAA0C;AAC1C,MAAM,WAAW,oBAAoB;IACnC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,UAAU,CAAC,EAAE,eAAe,CAAC;CAC9B;AAED,4EAA4E;AAC5E,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,iBAAiB,CAAC;CACjC;AAED,6EAA6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC;AAED,iEAAiE;AACjE,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CAC1D;AAED,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,iBAAiB,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,qBAAqB;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,sBAAsB;IACrC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,6GAA6G;AAC7G,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,iBAAiB,CAAC;IAE1B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,kBAAkB,CAAC;IAE7B,QAAQ,CAAC,EAAE,mBAAmB,CAAC;IAE/B,OAAO,CAAC,EAAE,kBAAkB,CAAC;IAE7B,UAAU,EAAE,qBAAqB,CAAC;IAClC,MAAM,EAAE,aAAa,CAAC;IAEtB,WAAW,EAAE,sBAAsB,CAAC;CACrC;AAED,6EAA6E;AAC7E,MAAM,WAAW,wBAAwB;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,iBAAiB,CAAC;IAC/B,UAAU,EAAE,qBAAqB,CAAC;IAClC,MAAM,EAAE,aAAa,CAAC;CACvB;AAMD,oEAAoE;AACpE,MAAM,WAAW,aAAa;IAC5B,WAAW,CAAC,KAAK,EAAE;QACjB,aAAa,EAAE,MAAM,CAAC;QACtB,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,UAAU,CAAC;QAClB,WAAW,EAAE,iBAAiB,CAAC;QAC/B,MAAM,EAAE,aAAa,CAAC;KACvB,GAAG,OAAO,CAAC;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAEpC,WAAW,CAAC,KAAK,EAAE;QACjB,aAAa,EAAE,MAAM,CAAC;QACtB,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElB,iBAAiB,CAAC,KAAK,EAAE;QACvB,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,iBAAiB,CAAC;KAChC,GAAG,OAAO,CAAC;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;CAC5C;AAED,uFAAuF;AACvF,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,iCAAiC;IACjC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACjF;AAED,gGAAgG;AAChG,MAAM,WAAW,kBAAkB;IACjC,yDAAyD;IACzD,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,8EAA8E;IAC9E,aAAa,CAAC,EAAE,SAAS,GAAG,KAAK,GAAG,OAAO,GAAG,SAAS,GAAG,KAAK,CAAC;CACjE;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,KAAK,EAAE;QACf,GAAG,EAAE,MAAM,CAAC;QACZ,SAAS,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACjC;AAED,+FAA+F;AAC/F,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC1B,oEAAoE;IACpE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,cAAc,CAAC,KAAK,EAAE;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,GAAG,EAAE,MAAM,CAAC;QACZ,OAAO,EAAE,kBAAkB,CAAC;KAC7B,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;CACrC;AAED,+EAA+E;AAC/E,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAC;CAC5D;AAED,gDAAgD;AAChD,MAAM,WAAW,qBAAqB;IACpC,cAAc,EAAE,YAAY,EAAE,CAAC;IAC/B,aAAa,EAAE,aAAa,CAAC;IAE7B,6EAA6E;IAC7E,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,kFAAkF;IAClF,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,iFAAiF;IACjF,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,gCAAgC;IAChC,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,8FAA8F;IAC9F,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0BAA0B;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,4DAA4D;IAC5D,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public types for @x12i/content-fetcher.
|
|
3
|
+
*
|
|
4
|
+
* These shapes are part of the package contract and mirror the package
|
|
5
|
+
* requirement document exactly. Downstream packages (extractor, normalizer)
|
|
6
|
+
* consume {@link FetchedArtifactReference}; do not rename or widen fields.
|
|
7
|
+
*/
|
|
8
|
+
export {};
|
|
9
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/** Concatenate byte chunks. */
|
|
2
|
+
export declare function concatBytes(chunks: Uint8Array[]): Uint8Array;
|
|
3
|
+
export declare function utf8Bytes(text: string): Uint8Array;
|
|
4
|
+
export interface BodyReadResult {
|
|
5
|
+
bytes: Uint8Array;
|
|
6
|
+
/** True when the stream produced more than `maxBytes` bytes. */
|
|
7
|
+
exceeded: boolean;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Read an async byte stream under a byte budget.
|
|
11
|
+
*
|
|
12
|
+
* - `maxBytes`: hard cap. Reading past it sets `exceeded` and stops iteration.
|
|
13
|
+
* - `prefixOnly` + `prefixBytes`: stop as soon as the prefix budget is met
|
|
14
|
+
* (used to sniff classification markers without downloading a full body).
|
|
15
|
+
*/
|
|
16
|
+
export declare function readBody(body: AsyncIterable<Uint8Array> | undefined, options: {
|
|
17
|
+
maxBytes: number;
|
|
18
|
+
prefixBytes?: number;
|
|
19
|
+
prefixOnly?: boolean;
|
|
20
|
+
}): Promise<BodyReadResult>;
|
|
21
|
+
//# sourceMappingURL=bytes.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bytes.d.ts","sourceRoot":"","sources":["../../src/util/bytes.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,wBAAgB,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,UAAU,CAU5D;AAID,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,CAElD;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,UAAU,CAAC;IAClB,gEAAgE;IAChE,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;;;;;GAMG;AACH,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,SAAS,EAC3C,OAAO,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,OAAO,CAAA;CAAE,GACxE,OAAO,CAAC,cAAc,CAAC,CAoBzB"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/** Concatenate byte chunks. */
|
|
2
|
+
export function concatBytes(chunks) {
|
|
3
|
+
let total = 0;
|
|
4
|
+
for (const chunk of chunks)
|
|
5
|
+
total += chunk.length;
|
|
6
|
+
const out = new Uint8Array(total);
|
|
7
|
+
let offset = 0;
|
|
8
|
+
for (const chunk of chunks) {
|
|
9
|
+
out.set(chunk, offset);
|
|
10
|
+
offset += chunk.length;
|
|
11
|
+
}
|
|
12
|
+
return out;
|
|
13
|
+
}
|
|
14
|
+
const encoder = new TextEncoder();
|
|
15
|
+
export function utf8Bytes(text) {
|
|
16
|
+
return encoder.encode(text);
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Read an async byte stream under a byte budget.
|
|
20
|
+
*
|
|
21
|
+
* - `maxBytes`: hard cap. Reading past it sets `exceeded` and stops iteration.
|
|
22
|
+
* - `prefixOnly` + `prefixBytes`: stop as soon as the prefix budget is met
|
|
23
|
+
* (used to sniff classification markers without downloading a full body).
|
|
24
|
+
*/
|
|
25
|
+
export async function readBody(body, options) {
|
|
26
|
+
const chunks = [];
|
|
27
|
+
let total = 0;
|
|
28
|
+
if (body) {
|
|
29
|
+
for await (const chunk of body) {
|
|
30
|
+
chunks.push(chunk);
|
|
31
|
+
total += chunk.length;
|
|
32
|
+
if (total > options.maxBytes) {
|
|
33
|
+
return { bytes: concatBytes(chunks), exceeded: true };
|
|
34
|
+
}
|
|
35
|
+
if (options.prefixOnly &&
|
|
36
|
+
options.prefixBytes !== undefined &&
|
|
37
|
+
total >= options.prefixBytes) {
|
|
38
|
+
break;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return { bytes: concatBytes(chunks), exceeded: false };
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=bytes.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bytes.js","sourceRoot":"","sources":["../../src/util/bytes.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,MAAM,UAAU,WAAW,CAAC,MAAoB;IAC9C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;IAClD,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;IACzB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAElC,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAQD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAA2C,EAC3C,OAAyE;IAEzE,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,IAAI,EAAE,CAAC;QACT,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YACtB,IAAI,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC7B,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YACxD,CAAC;YACD,IACE,OAAO,CAAC,UAAU;gBAClB,OAAO,CAAC,WAAW,KAAK,SAAS;gBACjC,KAAK,IAAI,OAAO,CAAC,WAAW,EAC5B,CAAC;gBACD,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AACzD,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { Sha256ContentHash } from "../types.js";
|
|
2
|
+
export declare function sha256Hex(bytes: Uint8Array): string;
|
|
3
|
+
export declare function sha256ContentHash(bytes: Uint8Array): Sha256ContentHash;
|
|
4
|
+
export declare function isSha256ContentHash(value: unknown): value is Sha256ContentHash;
|
|
5
|
+
//# sourceMappingURL=hash.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../../src/util/hash.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAErD,wBAAgB,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAEnD;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,UAAU,GAAG,iBAAiB,CAEtE;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,iBAAiB,CAE9E"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
export function sha256Hex(bytes) {
|
|
3
|
+
return createHash("sha256").update(bytes).digest("hex");
|
|
4
|
+
}
|
|
5
|
+
export function sha256ContentHash(bytes) {
|
|
6
|
+
return `sha256:${sha256Hex(bytes)}`;
|
|
7
|
+
}
|
|
8
|
+
export function isSha256ContentHash(value) {
|
|
9
|
+
return typeof value === "string" && /^sha256:[0-9a-f]{64}$/.test(value);
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=hash.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hash.js","sourceRoot":"","sources":["../../src/util/hash.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,MAAM,UAAU,SAAS,CAAC,KAAiB;IACzC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAiB;IACjD,OAAO,UAAU,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC1E,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ids.d.ts","sourceRoot":"","sources":["../../src/util/ids.ts"],"names":[],"mappings":"AAEA,wBAAgB,gBAAgB,IAAI,MAAM,CAEzC;AAED,wBAAgB,YAAY,IAAI,MAAM,CAErC"}
|
package/dist/util/ids.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ids.js","sourceRoot":"","sources":["../../src/util/ids.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,UAAU,gBAAgB;IAC9B,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAC/B,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { FetchRequest, FetchRequestResource } from "./types.js";
|
|
2
|
+
export type ResourceKind = "url" | "api_resource" | "feed_entry" | "archive_reference" | "licensed_resource" | "manual_file";
|
|
3
|
+
/**
|
|
4
|
+
* Runtime validation of a fetch request against the request contract.
|
|
5
|
+
* Throws InvalidFetchRequestError; returns a normalized deep copy.
|
|
6
|
+
*/
|
|
7
|
+
export declare function validateFetchRequest(input: unknown): FetchRequest;
|
|
8
|
+
export declare function resourceKindOf(resource: FetchRequestResource): ResourceKind;
|
|
9
|
+
//# sourceMappingURL=validate-request.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validate-request.d.ts","sourceRoot":"","sources":["../src/validate-request.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAWrE,MAAM,MAAM,YAAY,GACpB,KAAK,GACL,cAAc,GACd,YAAY,GACZ,mBAAmB,GACnB,mBAAmB,GACnB,aAAa,CAAC;AAelB;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY,CAwGjE;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,oBAAoB,GAAG,YAAY,CAO3E"}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { InvalidFetchRequestError } from "./errors.js";
|
|
2
|
+
import { isSha256ContentHash } from "./util/hash.js";
|
|
3
|
+
import { isMimeTypeLike, isNonEmptyString, isRecord, isStringArray, isUint8Array, } from "./runtime-utils.js";
|
|
4
|
+
const RESOURCE_KEYS = [
|
|
5
|
+
"url",
|
|
6
|
+
"apiResource",
|
|
7
|
+
"feedEntryUrl",
|
|
8
|
+
"archiveReference",
|
|
9
|
+
"licensedResourceId",
|
|
10
|
+
"manualFile",
|
|
11
|
+
];
|
|
12
|
+
function fail(code, message) {
|
|
13
|
+
throw new InvalidFetchRequestError(message, code);
|
|
14
|
+
}
|
|
15
|
+
function isAbsoluteUrl(raw) {
|
|
16
|
+
try {
|
|
17
|
+
const url = new URL(raw);
|
|
18
|
+
return url.hostname !== "";
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return false;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Runtime validation of a fetch request against the request contract.
|
|
26
|
+
* Throws InvalidFetchRequestError; returns a normalized deep copy.
|
|
27
|
+
*/
|
|
28
|
+
export function validateFetchRequest(input) {
|
|
29
|
+
if (!isRecord(input))
|
|
30
|
+
fail("request_not_object", "FetchRequest must be an object.");
|
|
31
|
+
if (!isNonEmptyString(input.requestId)) {
|
|
32
|
+
fail("missing_request_id", "requestId must be a non-empty string.");
|
|
33
|
+
}
|
|
34
|
+
if (!isNonEmptyString(input.sourceId)) {
|
|
35
|
+
fail("missing_source_id", "sourceId must be a non-empty string.");
|
|
36
|
+
}
|
|
37
|
+
if (!isRecord(input.resource)) {
|
|
38
|
+
fail("missing_resource", "resource must be an object.");
|
|
39
|
+
}
|
|
40
|
+
const resourceInput = input.resource;
|
|
41
|
+
const supplied = RESOURCE_KEYS.filter((key) => resourceInput[key] !== undefined);
|
|
42
|
+
if (supplied.length === 0) {
|
|
43
|
+
fail("resource_missing", "resource must supply exactly one resource reference.");
|
|
44
|
+
}
|
|
45
|
+
if (supplied.length > 1) {
|
|
46
|
+
fail("resource_ambiguous", `resource must supply exactly one resource reference, got: ${supplied.join(", ")}.`);
|
|
47
|
+
}
|
|
48
|
+
const kind = supplied[0];
|
|
49
|
+
const resource = {};
|
|
50
|
+
if (kind === "manualFile") {
|
|
51
|
+
const manual = input.resource.manualFile;
|
|
52
|
+
if (!isRecord(manual) ||
|
|
53
|
+
!isNonEmptyString(manual.fileName) ||
|
|
54
|
+
!isMimeTypeLike(manual.contentType) ||
|
|
55
|
+
!isUint8Array(manual.bytes) ||
|
|
56
|
+
manual.bytes.length === 0) {
|
|
57
|
+
fail("manual_file_invalid", "manualFile must supply a non-empty fileName, a MIME contentType, and at least one byte.");
|
|
58
|
+
}
|
|
59
|
+
resource.manualFile = {
|
|
60
|
+
fileName: manual.fileName,
|
|
61
|
+
contentType: manual.contentType.trim(),
|
|
62
|
+
bytes: manual.bytes,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
else {
|
|
66
|
+
const value = input.resource[kind];
|
|
67
|
+
if (!isNonEmptyString(value)) {
|
|
68
|
+
fail(`${kind}_invalid`, `resource.${kind} must be a non-empty string.`);
|
|
69
|
+
}
|
|
70
|
+
if (!isAbsoluteUrl(value)) {
|
|
71
|
+
fail(`${kind}_not_absolute`, `resource.${kind} must be an absolute URL with a host.`);
|
|
72
|
+
}
|
|
73
|
+
resource[kind] = value;
|
|
74
|
+
}
|
|
75
|
+
let expectedContentTypes;
|
|
76
|
+
if (input.expectedContentTypes !== undefined) {
|
|
77
|
+
if (!isStringArray(input.expectedContentTypes) || input.expectedContentTypes.length === 0) {
|
|
78
|
+
fail("expected_content_types_invalid", "expectedContentTypes must be a non-empty string array when provided.");
|
|
79
|
+
}
|
|
80
|
+
for (const entry of input.expectedContentTypes) {
|
|
81
|
+
if (!isMimeTypeLike(entry)) {
|
|
82
|
+
fail("expected_content_types_invalid", `expectedContentTypes contains an invalid MIME pattern: ${entry}.`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
expectedContentTypes = input.expectedContentTypes.map((e) => e.trim().toLowerCase());
|
|
86
|
+
}
|
|
87
|
+
let previousVersion;
|
|
88
|
+
if (input.previousVersion !== undefined) {
|
|
89
|
+
const pv = input.previousVersion;
|
|
90
|
+
if (!isRecord(pv))
|
|
91
|
+
fail("previous_version_invalid", "previousVersion must be an object when provided.");
|
|
92
|
+
if (pv.etag !== undefined && !isNonEmptyString(pv.etag)) {
|
|
93
|
+
fail("previous_version_invalid", "previousVersion.etag must be a non-empty string.");
|
|
94
|
+
}
|
|
95
|
+
if (pv.lastModified !== undefined && !isNonEmptyString(pv.lastModified)) {
|
|
96
|
+
fail("previous_version_invalid", "previousVersion.lastModified must be a non-empty string.");
|
|
97
|
+
}
|
|
98
|
+
if (pv.contentHash !== undefined) {
|
|
99
|
+
const normalized = String(pv.contentHash).toLowerCase();
|
|
100
|
+
if (!isSha256ContentHash(normalized)) {
|
|
101
|
+
fail("previous_version_invalid", "previousVersion.contentHash must match sha256:<64 lowercase hex>.");
|
|
102
|
+
}
|
|
103
|
+
previousVersion = {
|
|
104
|
+
...(pv.etag !== undefined ? { etag: pv.etag } : {}),
|
|
105
|
+
...(pv.lastModified !== undefined ? { lastModified: pv.lastModified } : {}),
|
|
106
|
+
...(pv.contentHash !== undefined ? { contentHash: normalized } : {}),
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
previousVersion = {
|
|
111
|
+
...(pv.etag !== undefined ? { etag: pv.etag } : {}),
|
|
112
|
+
...(pv.lastModified !== undefined ? { lastModified: pv.lastModified } : {}),
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return {
|
|
117
|
+
requestId: input.requestId,
|
|
118
|
+
sourceId: input.sourceId,
|
|
119
|
+
resource,
|
|
120
|
+
...(expectedContentTypes !== undefined ? { expectedContentTypes } : {}),
|
|
121
|
+
...(previousVersion !== undefined ? { previousVersion } : {}),
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
export function resourceKindOf(resource) {
|
|
125
|
+
if (resource.url !== undefined)
|
|
126
|
+
return "url";
|
|
127
|
+
if (resource.apiResource !== undefined)
|
|
128
|
+
return "api_resource";
|
|
129
|
+
if (resource.feedEntryUrl !== undefined)
|
|
130
|
+
return "feed_entry";
|
|
131
|
+
if (resource.archiveReference !== undefined)
|
|
132
|
+
return "archive_reference";
|
|
133
|
+
if (resource.licensedResourceId !== undefined)
|
|
134
|
+
return "licensed_resource";
|
|
135
|
+
return "manual_file";
|
|
136
|
+
}
|
|
137
|
+
//# sourceMappingURL=validate-request.js.map
|