@tangle-network/browser-agent-driver 0.29.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/brain/index.d.ts +15 -1
- package/dist/brain/index.d.ts.map +1 -1
- package/dist/brain/index.js +67 -5
- package/dist/brain/index.js.map +1 -1
- package/dist/cli-ack-patch.d.ts +39 -0
- package/dist/cli-ack-patch.d.ts.map +1 -0
- package/dist/cli-ack-patch.js +47 -0
- package/dist/cli-ack-patch.js.map +1 -0
- package/dist/cli-design-audit.d.ts +16 -0
- package/dist/cli-design-audit.d.ts.map +1 -1
- package/dist/cli-design-audit.js +232 -7
- package/dist/cli-design-audit.js.map +1 -1
- package/dist/cli-jobs.d.ts +13 -0
- package/dist/cli-jobs.d.ts.map +1 -0
- package/dist/cli-jobs.js +262 -0
- package/dist/cli-jobs.js.map +1 -0
- package/dist/cli-patterns.d.ts +26 -0
- package/dist/cli-patterns.d.ts.map +1 -0
- package/dist/cli-patterns.js +49 -0
- package/dist/cli-patterns.js.map +1 -0
- package/dist/cli-reports.d.ts +12 -0
- package/dist/cli-reports.d.ts.map +1 -0
- package/dist/cli-reports.js +97 -0
- package/dist/cli-reports.js.map +1 -0
- package/dist/cli-ui.d.ts.map +1 -1
- package/dist/cli-ui.js +5 -1
- package/dist/cli-ui.js.map +1 -1
- package/dist/cli.js +52 -2
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +4 -4
- package/dist/config.d.ts.map +1 -1
- package/dist/design/audit/attribution/aggregate.d.ts +16 -0
- package/dist/design/audit/attribution/aggregate.d.ts.map +1 -0
- package/dist/design/audit/attribution/aggregate.js +81 -0
- package/dist/design/audit/attribution/aggregate.js.map +1 -0
- package/dist/design/audit/attribution/store.d.ts +36 -0
- package/dist/design/audit/attribution/store.d.ts.map +1 -0
- package/dist/design/audit/attribution/store.js +108 -0
- package/dist/design/audit/attribution/store.js.map +1 -0
- package/dist/design/audit/attribution/types.d.ts +13 -0
- package/dist/design/audit/attribution/types.d.ts.map +1 -0
- package/dist/design/audit/attribution/types.js +10 -0
- package/dist/design/audit/attribution/types.js.map +1 -0
- package/dist/design/audit/classify-ensemble.d.ts +46 -0
- package/dist/design/audit/classify-ensemble.d.ts.map +1 -0
- package/dist/design/audit/classify-ensemble.js +207 -0
- package/dist/design/audit/classify-ensemble.js.map +1 -0
- package/dist/design/audit/ethics/check.d.ts +59 -0
- package/dist/design/audit/ethics/check.d.ts.map +1 -0
- package/dist/design/audit/ethics/check.js +137 -0
- package/dist/design/audit/ethics/check.js.map +1 -0
- package/dist/design/audit/ethics/loader.d.ts +21 -0
- package/dist/design/audit/ethics/loader.d.ts.map +1 -0
- package/dist/design/audit/ethics/loader.js +207 -0
- package/dist/design/audit/ethics/loader.js.map +1 -0
- package/dist/design/audit/ethics/rules/finance.yaml +37 -0
- package/dist/design/audit/ethics/rules/kids.yaml +37 -0
- package/dist/design/audit/ethics/rules/legal.yaml +35 -0
- package/dist/design/audit/ethics/rules/medical.yaml +36 -0
- package/dist/design/audit/evaluate.d.ts +69 -0
- package/dist/design/audit/evaluate.d.ts.map +1 -1
- package/dist/design/audit/evaluate.js +417 -55
- package/dist/design/audit/evaluate.js.map +1 -1
- package/dist/design/audit/first-principles-mode.d.ts +42 -0
- package/dist/design/audit/first-principles-mode.d.ts.map +1 -0
- package/dist/design/audit/first-principles-mode.js +77 -0
- package/dist/design/audit/first-principles-mode.js.map +1 -0
- package/dist/design/audit/modality/android.d.ts +14 -0
- package/dist/design/audit/modality/android.d.ts.map +1 -0
- package/dist/design/audit/modality/android.js +18 -0
- package/dist/design/audit/modality/android.js.map +1 -0
- package/dist/design/audit/modality/html.d.ts +22 -0
- package/dist/design/audit/modality/html.d.ts.map +1 -0
- package/dist/design/audit/modality/html.js +51 -0
- package/dist/design/audit/modality/html.js.map +1 -0
- package/dist/design/audit/modality/index.d.ts +8 -0
- package/dist/design/audit/modality/index.d.ts.map +1 -0
- package/dist/design/audit/modality/index.js +15 -0
- package/dist/design/audit/modality/index.js.map +1 -0
- package/dist/design/audit/modality/ios.d.ts +16 -0
- package/dist/design/audit/modality/ios.d.ts.map +1 -0
- package/dist/design/audit/modality/ios.js +19 -0
- package/dist/design/audit/modality/ios.js.map +1 -0
- package/dist/design/audit/modality/types.d.ts +9 -0
- package/dist/design/audit/modality/types.d.ts.map +1 -0
- package/dist/design/audit/modality/types.js +9 -0
- package/dist/design/audit/modality/types.js.map +1 -0
- package/dist/design/audit/patches/index.d.ts +8 -0
- package/dist/design/audit/patches/index.d.ts.map +1 -0
- package/dist/design/audit/patches/index.js +5 -0
- package/dist/design/audit/patches/index.js.map +1 -0
- package/dist/design/audit/patches/parse.d.ts +29 -0
- package/dist/design/audit/patches/parse.d.ts.map +1 -0
- package/dist/design/audit/patches/parse.js +153 -0
- package/dist/design/audit/patches/parse.js.map +1 -0
- package/dist/design/audit/patches/render.d.ts +20 -0
- package/dist/design/audit/patches/render.d.ts.map +1 -0
- package/dist/design/audit/patches/render.js +55 -0
- package/dist/design/audit/patches/render.js.map +1 -0
- package/dist/design/audit/patches/severity-enforcement.d.ts +23 -0
- package/dist/design/audit/patches/severity-enforcement.d.ts.map +1 -0
- package/dist/design/audit/patches/severity-enforcement.js +42 -0
- package/dist/design/audit/patches/severity-enforcement.js.map +1 -0
- package/dist/design/audit/patches/validate.d.ts +34 -0
- package/dist/design/audit/patches/validate.d.ts.map +1 -0
- package/dist/design/audit/patches/validate.js +53 -0
- package/dist/design/audit/patches/validate.js.map +1 -0
- package/dist/design/audit/patterns/match.d.ts +24 -0
- package/dist/design/audit/patterns/match.d.ts.map +1 -0
- package/dist/design/audit/patterns/match.js +48 -0
- package/dist/design/audit/patterns/match.js.map +1 -0
- package/dist/design/audit/patterns/mine.d.ts +31 -0
- package/dist/design/audit/patterns/mine.d.ts.map +1 -0
- package/dist/design/audit/patterns/mine.js +35 -0
- package/dist/design/audit/patterns/mine.js.map +1 -0
- package/dist/design/audit/patterns/store.d.ts +14 -0
- package/dist/design/audit/patterns/store.d.ts.map +1 -0
- package/dist/design/audit/patterns/store.js +53 -0
- package/dist/design/audit/patterns/store.js.map +1 -0
- package/dist/design/audit/patterns/types.d.ts +53 -0
- package/dist/design/audit/patterns/types.d.ts.map +1 -0
- package/dist/design/audit/patterns/types.js +13 -0
- package/dist/design/audit/patterns/types.js.map +1 -0
- package/dist/design/audit/pipeline.d.ts +26 -0
- package/dist/design/audit/pipeline.d.ts.map +1 -1
- package/dist/design/audit/pipeline.js +131 -4
- package/dist/design/audit/pipeline.js.map +1 -1
- package/dist/design/audit/rubric/anchor-loader.d.ts +44 -0
- package/dist/design/audit/rubric/anchor-loader.d.ts.map +1 -0
- package/dist/design/audit/rubric/anchor-loader.js +210 -0
- package/dist/design/audit/rubric/anchor-loader.js.map +1 -0
- package/dist/design/audit/rubric/anchors/blog.yaml +33 -0
- package/dist/design/audit/rubric/anchors/dashboard.yaml +35 -0
- package/dist/design/audit/rubric/anchors/docs.yaml +35 -0
- package/dist/design/audit/rubric/anchors/ecommerce.yaml +33 -0
- package/dist/design/audit/rubric/anchors/marketing.yaml +34 -0
- package/dist/design/audit/rubric/anchors/saas-app.yaml +38 -0
- package/dist/design/audit/rubric/anchors/social.yaml +33 -0
- package/dist/design/audit/rubric/anchors/tool.yaml +33 -0
- package/dist/design/audit/rubric/anchors/utility.yaml +33 -0
- package/dist/design/audit/rubric/fragments/audience-clinician.md +42 -0
- package/dist/design/audit/rubric/fragments/audience-developer.md +40 -0
- package/dist/design/audit/rubric/fragments/audience-kids.md +35 -0
- package/dist/design/audit/rubric/fragments/audience-vulnerability-minor-facing.md +37 -0
- package/dist/design/audit/rubric/fragments/first-principles.md +65 -0
- package/dist/design/audit/rubric/fragments/modality-mobile.md +39 -0
- package/dist/design/audit/rubric/fragments/modality-tablet.md +35 -0
- package/dist/design/audit/rubric/fragments/regulatory-coppa.md +32 -0
- package/dist/design/audit/rubric/fragments/regulatory-gdpr.md +33 -0
- package/dist/design/audit/rubric/fragments/regulatory-hipaa.md +36 -0
- package/dist/design/audit/rubric/fragments/universal-product-intent.md +48 -0
- package/dist/design/audit/rubric/loader.d.ts +26 -6
- package/dist/design/audit/rubric/loader.d.ts.map +1 -1
- package/dist/design/audit/rubric/loader.js +77 -27
- package/dist/design/audit/rubric/loader.js.map +1 -1
- package/dist/design/audit/rubric/rollup-weights.d.ts +25 -0
- package/dist/design/audit/rubric/rollup-weights.d.ts.map +1 -0
- package/dist/design/audit/rubric/rollup-weights.js +51 -0
- package/dist/design/audit/rubric/rollup-weights.js.map +1 -0
- package/dist/design/audit/types.d.ts +28 -0
- package/dist/design/audit/types.d.ts.map +1 -1
- package/dist/design/audit/v2/build-result.d.ts +47 -0
- package/dist/design/audit/v2/build-result.d.ts.map +1 -0
- package/dist/design/audit/v2/build-result.js +163 -0
- package/dist/design/audit/v2/build-result.js.map +1 -0
- package/dist/design/audit/v2/score.d.ts +49 -0
- package/dist/design/audit/v2/score.d.ts.map +1 -0
- package/dist/design/audit/v2/score.js +211 -0
- package/dist/design/audit/v2/score.js.map +1 -0
- package/dist/design/audit/v2/types.d.ts +431 -0
- package/dist/design/audit/v2/types.d.ts.map +1 -0
- package/dist/design/audit/v2/types.js +26 -0
- package/dist/design/audit/v2/types.js.map +1 -0
- package/dist/discover/index.d.ts +16 -0
- package/dist/discover/index.d.ts.map +1 -0
- package/dist/discover/index.js +25 -0
- package/dist/discover/index.js.map +1 -0
- package/dist/discover/wayback.d.ts +49 -0
- package/dist/discover/wayback.d.ts.map +1 -0
- package/dist/discover/wayback.js +123 -0
- package/dist/discover/wayback.js.map +1 -0
- package/dist/jobs/anti-bot.d.ts +19 -0
- package/dist/jobs/anti-bot.d.ts.map +1 -0
- package/dist/jobs/anti-bot.js +64 -0
- package/dist/jobs/anti-bot.js.map +1 -0
- package/dist/jobs/cost-estimate.d.ts +12 -0
- package/dist/jobs/cost-estimate.d.ts.map +1 -0
- package/dist/jobs/cost-estimate.js +21 -0
- package/dist/jobs/cost-estimate.js.map +1 -0
- package/dist/jobs/cost-history.d.ts +19 -0
- package/dist/jobs/cost-history.d.ts.map +1 -0
- package/dist/jobs/cost-history.js +42 -0
- package/dist/jobs/cost-history.js.map +1 -0
- package/dist/jobs/index.d.ts +31 -0
- package/dist/jobs/index.d.ts.map +1 -0
- package/dist/jobs/index.js +37 -0
- package/dist/jobs/index.js.map +1 -0
- package/dist/jobs/orchestrator.d.ts +42 -0
- package/dist/jobs/orchestrator.d.ts.map +1 -0
- package/dist/jobs/orchestrator.js +282 -0
- package/dist/jobs/orchestrator.js.map +1 -0
- package/dist/jobs/queue.d.ts +45 -0
- package/dist/jobs/queue.d.ts.map +1 -0
- package/dist/jobs/queue.js +114 -0
- package/dist/jobs/queue.js.map +1 -0
- package/dist/jobs/retry.d.ts +36 -0
- package/dist/jobs/retry.d.ts.map +1 -0
- package/dist/jobs/retry.js +77 -0
- package/dist/jobs/retry.js.map +1 -0
- package/dist/jobs/store.d.ts +31 -0
- package/dist/jobs/store.d.ts.map +1 -0
- package/dist/jobs/store.js +92 -0
- package/dist/jobs/store.js.map +1 -0
- package/dist/jobs/types.d.ts +106 -0
- package/dist/jobs/types.d.ts.map +1 -0
- package/dist/jobs/types.js +13 -0
- package/dist/jobs/types.js.map +1 -0
- package/dist/provider-defaults.d.ts +1 -1
- package/dist/provider-defaults.d.ts.map +1 -1
- package/dist/provider-defaults.js +8 -0
- package/dist/provider-defaults.js.map +1 -1
- package/dist/reports/aggregate.d.ts +38 -0
- package/dist/reports/aggregate.d.ts.map +1 -0
- package/dist/reports/aggregate.js +145 -0
- package/dist/reports/aggregate.js.map +1 -0
- package/dist/reports/index.d.ts +21 -0
- package/dist/reports/index.d.ts.map +1 -0
- package/dist/reports/index.js +15 -0
- package/dist/reports/index.js.map +1 -0
- package/dist/reports/narrate.d.ts +19 -0
- package/dist/reports/narrate.d.ts.map +1 -0
- package/dist/reports/narrate.js +40 -0
- package/dist/reports/narrate.js.map +1 -0
- package/dist/reports/templates.d.ts +43 -0
- package/dist/reports/templates.d.ts.map +1 -0
- package/dist/reports/templates.js +200 -0
- package/dist/reports/templates.js.map +1 -0
- package/dist/reports/tokens.d.ts +77 -0
- package/dist/reports/tokens.d.ts.map +1 -0
- package/dist/reports/tokens.js +88 -0
- package/dist/reports/tokens.js.map +1 -0
- package/dist/reports/tools.d.ts +76 -0
- package/dist/reports/tools.d.ts.map +1 -0
- package/dist/reports/tools.js +205 -0
- package/dist/reports/tools.js.map +1 -0
- package/dist/reports/types.d.ts +51 -0
- package/dist/reports/types.d.ts.map +1 -0
- package/dist/reports/types.js +9 -0
- package/dist/reports/types.js.map +1 -0
- package/dist/supervisor/critic.d.ts +1 -1
- package/dist/supervisor/critic.d.ts.map +1 -1
- package/dist/supervisor/critic.js +50 -5
- package/dist/supervisor/critic.js.map +1 -1
- package/dist/telemetry/client.d.ts +42 -0
- package/dist/telemetry/client.d.ts.map +1 -0
- package/dist/telemetry/client.js +203 -0
- package/dist/telemetry/client.js.map +1 -0
- package/dist/telemetry/hash.d.ts +4 -0
- package/dist/telemetry/hash.d.ts.map +1 -0
- package/dist/telemetry/hash.js +7 -0
- package/dist/telemetry/hash.js.map +1 -0
- package/dist/telemetry/index.d.ts +6 -0
- package/dist/telemetry/index.d.ts.map +1 -0
- package/dist/telemetry/index.js +5 -0
- package/dist/telemetry/index.js.map +1 -0
- package/dist/telemetry/schema.d.ts +85 -0
- package/dist/telemetry/schema.d.ts.map +1 -0
- package/dist/telemetry/schema.js +13 -0
- package/dist/telemetry/schema.js.map +1 -0
- package/dist/telemetry/sink.d.ts +46 -0
- package/dist/telemetry/sink.d.ts.map +1 -0
- package/dist/telemetry/sink.js +97 -0
- package/dist/telemetry/sink.js.map +1 -0
- package/dist/types.d.ts +5 -5
- package/dist/types.d.ts.map +1 -1
- package/package.json +5 -1
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Design audit v2 — type contract for the 8-layer architecture.
|
|
3
|
+
*
|
|
4
|
+
* RFC: docs/rfc/design-audit-world-class.md
|
|
5
|
+
*
|
|
6
|
+
* This file is the stable contract that every layer's implementation
|
|
7
|
+
* builds against. It exists to let parallel implementation work proceed
|
|
8
|
+
* without diverging interfaces. Editing this file mid-build is a coordinated
|
|
9
|
+
* change; layers must update in lockstep.
|
|
10
|
+
*
|
|
11
|
+
* Invariants enforced by this contract:
|
|
12
|
+
* - Every score is a `DimensionScore` with `range` + `confidence`. No bare numbers.
|
|
13
|
+
* - Every finding with `severity in ['major','critical']` MUST have ≥1 `Patch`.
|
|
14
|
+
* - Every patch has both `target` (what changes) and `testThatProves` (how we verify).
|
|
15
|
+
* - Every classification carries explicit `ensembleConfidence` and `signalsAgreed`.
|
|
16
|
+
* - Every audit run can write a `PatchApplication` event for post-hoc attribution.
|
|
17
|
+
* - Pattern, ethics, modality types compose cleanly via shared `AppliesWhen`.
|
|
18
|
+
*/
|
|
19
|
+
export const DIMENSIONS = [
|
|
20
|
+
'product_intent',
|
|
21
|
+
'visual_craft',
|
|
22
|
+
'trust_clarity',
|
|
23
|
+
'workflow',
|
|
24
|
+
'content_ia',
|
|
25
|
+
];
|
|
26
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/design/audit/v2/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AA4BH,MAAM,CAAC,MAAM,UAAU,GAAyB;IAC9C,gBAAgB;IAChB,cAAc;IACd,eAAe;IACf,UAAU;IACV,YAAY;CACJ,CAAA"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Discoverers — turn a job's `discover` spec into a flat list of audit targets.
|
|
3
|
+
*
|
|
4
|
+
* Two sources:
|
|
5
|
+
* list — explicit URLs (passthrough).
|
|
6
|
+
* wayback — expand each URL into N historical snapshots via the CDX API.
|
|
7
|
+
*/
|
|
8
|
+
import type { DiscoverSpec, JobTarget } from '../jobs/types.js';
|
|
9
|
+
export { discoverWaybackSnapshots, expandWaybackTargets, snapshotUrl, sampleEvenly, parseCdxRows, cdxStampToIso } from './wayback.js';
|
|
10
|
+
export type { WaybackOptions } from './wayback.js';
|
|
11
|
+
export interface DiscoverOptions {
|
|
12
|
+
/** Injected fetch (tests). */
|
|
13
|
+
fetch?: typeof fetch;
|
|
14
|
+
}
|
|
15
|
+
export declare function discoverTargets(spec: DiscoverSpec, opts?: DiscoverOptions): Promise<JobTarget[]>;
|
|
16
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/discover/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAG/D,OAAO,EAAE,wBAAwB,EAAE,oBAAoB,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AACrI,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAElD,MAAM,WAAW,eAAe;IAC9B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,OAAO,KAAK,CAAA;CACrB;AAED,wBAAsB,eAAe,CAAC,IAAI,EAAE,YAAY,EAAE,IAAI,GAAE,eAAoB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAc1G"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Discoverers — turn a job's `discover` spec into a flat list of audit targets.
|
|
3
|
+
*
|
|
4
|
+
* Two sources:
|
|
5
|
+
* list — explicit URLs (passthrough).
|
|
6
|
+
* wayback — expand each URL into N historical snapshots via the CDX API.
|
|
7
|
+
*/
|
|
8
|
+
import { expandWaybackTargets } from './wayback.js';
|
|
9
|
+
export { discoverWaybackSnapshots, expandWaybackTargets, snapshotUrl, sampleEvenly, parseCdxRows, cdxStampToIso } from './wayback.js';
|
|
10
|
+
export async function discoverTargets(spec, opts = {}) {
|
|
11
|
+
if (spec.source === 'list') {
|
|
12
|
+
return spec.urls.map(url => ({ url }));
|
|
13
|
+
}
|
|
14
|
+
if (spec.source === 'wayback') {
|
|
15
|
+
const wb = {
|
|
16
|
+
count: spec.snapshotsPerUrl,
|
|
17
|
+
since: spec.since,
|
|
18
|
+
until: spec.until,
|
|
19
|
+
fetch: opts.fetch,
|
|
20
|
+
};
|
|
21
|
+
return expandWaybackTargets(spec.urls, wb);
|
|
22
|
+
}
|
|
23
|
+
throw new Error(`discover: unsupported source "${spec.source}"`);
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/discover/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,EAAE,oBAAoB,EAAuB,MAAM,cAAc,CAAA;AAExE,OAAO,EAAE,wBAAwB,EAAE,oBAAoB,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAQrI,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAkB,EAAE,OAAwB,EAAE;IAClF,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;QAC3B,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;IACxC,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,EAAE,GAAmB;YACzB,KAAK,EAAE,IAAI,CAAC,eAAe;YAC3B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAA;QACD,OAAO,oBAAoB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IAC5C,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,iCAAkC,IAAqB,CAAC,MAAM,GAAG,CAAC,CAAA;AACpF,CAAC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wayback Machine snapshot discoverer (CDX API).
|
|
3
|
+
*
|
|
4
|
+
* Uses archive.org's CDX server to list captures of a URL, then samples
|
|
5
|
+
* `count` snapshots evenly across the time range. The CDX API is rate-limited
|
|
6
|
+
* (~15 req/s) but for the small N we use here we never approach the cap.
|
|
7
|
+
*
|
|
8
|
+
* CDX response shape:
|
|
9
|
+
* [["urlkey","timestamp","original","mimetype","statuscode","digest","length"],
|
|
10
|
+
* ["com,stripe)/", "20100101120000", "https://stripe.com/", "text/html", "200", "...", "1234"],
|
|
11
|
+
* ...]
|
|
12
|
+
*
|
|
13
|
+
* Snapshot URL = `https://web.archive.org/web/<timestamp>/<original>`.
|
|
14
|
+
*/
|
|
15
|
+
import type { JobTarget } from '../jobs/types.js';
|
|
16
|
+
export interface WaybackOptions {
|
|
17
|
+
/** Snapshots per URL. Default 5. */
|
|
18
|
+
count?: number;
|
|
19
|
+
/** Lower-bound capture date (ISO 8601 — e.g. "2010-01-01"). */
|
|
20
|
+
since?: string;
|
|
21
|
+
/** Upper-bound capture date. */
|
|
22
|
+
until?: string;
|
|
23
|
+
/** Limit only HTTP 200 captures. Default true — 4xx/5xx snapshots aren't auditable. */
|
|
24
|
+
status200Only?: boolean;
|
|
25
|
+
/** Injected fetch (tests). Defaults to globalThis.fetch (Node 18+). */
|
|
26
|
+
fetch?: typeof fetch;
|
|
27
|
+
}
|
|
28
|
+
interface CdxRow {
|
|
29
|
+
urlkey: string;
|
|
30
|
+
timestamp: string;
|
|
31
|
+
original: string;
|
|
32
|
+
mimetype: string;
|
|
33
|
+
statuscode: string;
|
|
34
|
+
digest: string;
|
|
35
|
+
length: string;
|
|
36
|
+
}
|
|
37
|
+
/** Convert CDX timestamp "20100101120000" to ISO datetime. */
|
|
38
|
+
export declare function cdxStampToIso(stamp: string): string;
|
|
39
|
+
export declare function snapshotUrl(timestamp: string, original: string): string;
|
|
40
|
+
/** Sample `count` evenly-spaced rows from a sorted list. Always includes first and last. */
|
|
41
|
+
export declare function sampleEvenly<T>(rows: T[], count: number): T[];
|
|
42
|
+
/** Parse the CDX JSON response (first row is a header). */
|
|
43
|
+
export declare function parseCdxRows(json: unknown): CdxRow[];
|
|
44
|
+
/** Discover N evenly-spaced wayback snapshots for a URL. */
|
|
45
|
+
export declare function discoverWaybackSnapshots(url: string, opts?: WaybackOptions): Promise<JobTarget[]>;
|
|
46
|
+
/** Expand a list of seed URLs into wayback targets, in parallel. */
|
|
47
|
+
export declare function expandWaybackTargets(urls: string[], opts?: WaybackOptions): Promise<JobTarget[]>;
|
|
48
|
+
export {};
|
|
49
|
+
//# sourceMappingURL=wayback.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wayback.d.ts","sourceRoot":"","sources":["../../src/discover/wayback.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAIjD,MAAM,WAAW,cAAc;IAC7B,oCAAoC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,gCAAgC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,uFAAuF;IACvF,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,uEAAuE;IACvE,KAAK,CAAC,EAAE,OAAO,KAAK,CAAA;CACrB;AAED,UAAU,MAAM;IACd,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;CACf;AAcD,8DAA8D;AAC9D,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CASnD;AAED,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEvE;AAED,4FAA4F;AAC5F,wBAAgB,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,CAAC,EAAE,CAU7D;AAED,2DAA2D;AAC3D,wBAAgB,YAAY,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,EAAE,CAkBpD;AAED,4DAA4D;AAC5D,wBAAsB,wBAAwB,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE,cAAmB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAmC3G;AAED,oEAAoE;AACpE,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE,cAAmB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAG1G"}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wayback Machine snapshot discoverer (CDX API).
|
|
3
|
+
*
|
|
4
|
+
* Uses archive.org's CDX server to list captures of a URL, then samples
|
|
5
|
+
* `count` snapshots evenly across the time range. The CDX API is rate-limited
|
|
6
|
+
* (~15 req/s) but for the small N we use here we never approach the cap.
|
|
7
|
+
*
|
|
8
|
+
* CDX response shape:
|
|
9
|
+
* [["urlkey","timestamp","original","mimetype","statuscode","digest","length"],
|
|
10
|
+
* ["com,stripe)/", "20100101120000", "https://stripe.com/", "text/html", "200", "...", "1234"],
|
|
11
|
+
* ...]
|
|
12
|
+
*
|
|
13
|
+
* Snapshot URL = `https://web.archive.org/web/<timestamp>/<original>`.
|
|
14
|
+
*/
|
|
15
|
+
const CDX_ENDPOINT = 'https://web.archive.org/cdx/search/cdx';
|
|
16
|
+
const DEFAULT_COUNT = 5;
|
|
17
|
+
/** Convert ISO date "2010-01-01" to CDX yyyymmddhhmmss. */
|
|
18
|
+
function isoToCdxStamp(iso, end = false) {
|
|
19
|
+
const d = new Date(iso);
|
|
20
|
+
if (Number.isNaN(d.getTime()))
|
|
21
|
+
throw new Error(`wayback: invalid date "${iso}"`);
|
|
22
|
+
const y = d.getUTCFullYear();
|
|
23
|
+
const m = String(d.getUTCMonth() + 1).padStart(2, '0');
|
|
24
|
+
const day = String(d.getUTCDate()).padStart(2, '0');
|
|
25
|
+
return end ? `${y}${m}${day}235959` : `${y}${m}${day}000000`;
|
|
26
|
+
}
|
|
27
|
+
/** Convert CDX timestamp "20100101120000" to ISO datetime. */
|
|
28
|
+
export function cdxStampToIso(stamp) {
|
|
29
|
+
if (stamp.length < 14)
|
|
30
|
+
throw new Error(`wayback: malformed CDX timestamp "${stamp}"`);
|
|
31
|
+
const y = stamp.slice(0, 4);
|
|
32
|
+
const m = stamp.slice(4, 6);
|
|
33
|
+
const d = stamp.slice(6, 8);
|
|
34
|
+
const hh = stamp.slice(8, 10);
|
|
35
|
+
const mm = stamp.slice(10, 12);
|
|
36
|
+
const ss = stamp.slice(12, 14);
|
|
37
|
+
return `${y}-${m}-${d}T${hh}:${mm}:${ss}Z`;
|
|
38
|
+
}
|
|
39
|
+
export function snapshotUrl(timestamp, original) {
|
|
40
|
+
return `https://web.archive.org/web/${timestamp}/${original}`;
|
|
41
|
+
}
|
|
42
|
+
/** Sample `count` evenly-spaced rows from a sorted list. Always includes first and last. */
|
|
43
|
+
export function sampleEvenly(rows, count) {
|
|
44
|
+
if (rows.length <= count)
|
|
45
|
+
return [...rows];
|
|
46
|
+
if (count <= 0)
|
|
47
|
+
return [];
|
|
48
|
+
if (count === 1)
|
|
49
|
+
return [rows[Math.floor((rows.length - 1) / 2)]];
|
|
50
|
+
const out = [];
|
|
51
|
+
const step = (rows.length - 1) / (count - 1);
|
|
52
|
+
for (let i = 0; i < count; i++) {
|
|
53
|
+
out.push(rows[Math.round(i * step)]);
|
|
54
|
+
}
|
|
55
|
+
return out;
|
|
56
|
+
}
|
|
57
|
+
/** Parse the CDX JSON response (first row is a header). */
|
|
58
|
+
export function parseCdxRows(json) {
|
|
59
|
+
if (!Array.isArray(json) || json.length < 2)
|
|
60
|
+
return [];
|
|
61
|
+
// Skip header row.
|
|
62
|
+
const rows = json.slice(1);
|
|
63
|
+
const out = [];
|
|
64
|
+
for (const r of rows) {
|
|
65
|
+
if (!Array.isArray(r) || r.length < 7)
|
|
66
|
+
continue;
|
|
67
|
+
out.push({
|
|
68
|
+
urlkey: String(r[0]),
|
|
69
|
+
timestamp: String(r[1]),
|
|
70
|
+
original: String(r[2]),
|
|
71
|
+
mimetype: String(r[3]),
|
|
72
|
+
statuscode: String(r[4]),
|
|
73
|
+
digest: String(r[5]),
|
|
74
|
+
length: String(r[6]),
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
return out;
|
|
78
|
+
}
|
|
79
|
+
/** Discover N evenly-spaced wayback snapshots for a URL. */
|
|
80
|
+
export async function discoverWaybackSnapshots(url, opts = {}) {
|
|
81
|
+
const fetchImpl = opts.fetch ?? globalThis.fetch;
|
|
82
|
+
if (!fetchImpl)
|
|
83
|
+
throw new Error('wayback: no fetch implementation available');
|
|
84
|
+
const count = opts.count ?? DEFAULT_COUNT;
|
|
85
|
+
const params = new URLSearchParams({
|
|
86
|
+
url,
|
|
87
|
+
output: 'json',
|
|
88
|
+
// collapse=timestamp:6 dedupes to one capture per month (yyyymm = 6 chars).
|
|
89
|
+
// Without this, CDX returns every capture in the window — which for popular
|
|
90
|
+
// sites is tens of thousands and silently skews `sampleEvenly` if combined
|
|
91
|
+
// with a `limit`. With the collapse, the row count is bounded by the
|
|
92
|
+
// window length in months, so we don't need a limit.
|
|
93
|
+
collapse: 'timestamp:6',
|
|
94
|
+
});
|
|
95
|
+
if (opts.since)
|
|
96
|
+
params.set('from', isoToCdxStamp(opts.since));
|
|
97
|
+
if (opts.until)
|
|
98
|
+
params.set('to', isoToCdxStamp(opts.until, true));
|
|
99
|
+
if (opts.status200Only !== false) {
|
|
100
|
+
params.set('filter', 'statuscode:200');
|
|
101
|
+
}
|
|
102
|
+
const resp = await fetchImpl(`${CDX_ENDPOINT}?${params.toString()}`);
|
|
103
|
+
if (!resp.ok) {
|
|
104
|
+
throw new Error(`wayback: CDX returned ${resp.status} for ${url}`);
|
|
105
|
+
}
|
|
106
|
+
const json = await resp.json();
|
|
107
|
+
const rows = parseCdxRows(json);
|
|
108
|
+
if (rows.length === 0)
|
|
109
|
+
return [];
|
|
110
|
+
// CDX returns chronological order by default. Sample evenly across.
|
|
111
|
+
const sampled = sampleEvenly(rows, count);
|
|
112
|
+
return sampled.map(r => ({
|
|
113
|
+
url,
|
|
114
|
+
snapshotUrl: snapshotUrl(r.timestamp, r.original),
|
|
115
|
+
capturedAt: cdxStampToIso(r.timestamp),
|
|
116
|
+
}));
|
|
117
|
+
}
|
|
118
|
+
/** Expand a list of seed URLs into wayback targets, in parallel. */
|
|
119
|
+
export async function expandWaybackTargets(urls, opts = {}) {
|
|
120
|
+
const all = await Promise.all(urls.map(u => discoverWaybackSnapshots(u, opts).catch(() => [])));
|
|
121
|
+
return all.flat();
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=wayback.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wayback.js","sourceRoot":"","sources":["../../src/discover/wayback.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAIH,MAAM,YAAY,GAAG,wCAAwC,CAAA;AAyB7D,MAAM,aAAa,GAAG,CAAC,CAAA;AAEvB,2DAA2D;AAC3D,SAAS,aAAa,CAAC,GAAW,EAAE,GAAG,GAAG,KAAK;IAC7C,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAA;IACvB,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,GAAG,CAAC,CAAA;IAChF,MAAM,CAAC,GAAG,CAAC,CAAC,cAAc,EAAE,CAAA;IAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;IACtD,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;IACnD,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,QAAQ,CAAA;AAC9D,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,IAAI,KAAK,CAAC,MAAM,GAAG,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,KAAK,GAAG,CAAC,CAAA;IACrF,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC3B,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC3B,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC3B,MAAM,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAC7B,MAAM,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;IAC9B,MAAM,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;IAC9B,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAA;AAC5C,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,SAAiB,EAAE,QAAgB;IAC7D,OAAO,+BAA+B,SAAS,IAAI,QAAQ,EAAE,CAAA;AAC/D,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,YAAY,CAAI,IAAS,EAAE,KAAa;IACtD,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK;QAAE,OAAO,CAAC,GAAG,IAAI,CAAC,CAAA;IAC1C,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,EAAE,CAAA;IACzB,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IACjE,MAAM,GAAG,GAAQ,EAAE,CAAA;IACnB,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;IAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IACtC,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,YAAY,CAAC,IAAa;IACxC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAA;IACtD,mBAAmB;IACnB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAc,CAAA;IACvC,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,SAAQ;QAC/C,GAAG,CAAC,IAAI,CAAC;YACP,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACtB,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACtB,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SACrB,CAAC,CAAA;IACJ,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,4DAA4D;AAC5D,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,GAAW,EAAE,OAAuB,EAAE;IACnF,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,IAAI,UAAU,CAAC,KAAK,CAAA;IAChD,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;IAE7E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,aAAa,CAAA;IACzC,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;QACjC,GAAG;QACH,MAAM,EAAE,MAAM;QACd,4EAA4E;QAC5E,4EAA4E;QAC5E,2EAA2E;QAC3E,qEAAqE;QACrE,qDAAqD;QACrD,QAAQ,EAAE,aAAa;KACxB,CAAC,CAAA;IACF,IAAI,IAAI,CAAC,KAAK;QAAE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;IAC7D,IAAI,IAAI,CAAC,KAAK;QAAE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAA;IACjE,IAAI,IAAI,CAAC,aAAa,KAAK,KAAK,EAAE,CAAC;QACjC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAA;IACxC,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,YAAY,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IACpE,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,CAAC,MAAM,QAAQ,GAAG,EAAE,CAAC,CAAA;IACpE,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAa,CAAA;IACzC,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;IAC/B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IAChC,oEAAoE;IACpE,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACzC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvB,GAAG;QACH,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC;QACjD,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC;KACvC,CAAC,CAAC,CAAA;AACL,CAAC;AAED,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAc,EAAE,OAAuB,EAAE;IAClF,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,wBAAwB,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAiB,CAAC,CAAC,CAAC,CAAA;IAC9G,OAAO,GAAG,CAAC,IAAI,EAAE,CAAA;AACnB,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Anti-bot / blocked-page detection. Pure pattern match against an audit's
|
|
3
|
+
* report.json — we propagate the existing audit signals rather than re-running
|
|
4
|
+
* inference.
|
|
5
|
+
*
|
|
6
|
+
* Returns a reason string when blocked (so the job can carry it through to
|
|
7
|
+
* the result envelope), else null.
|
|
8
|
+
*/
|
|
9
|
+
export interface BlockSignals {
|
|
10
|
+
title?: string;
|
|
11
|
+
intent?: string;
|
|
12
|
+
type?: string;
|
|
13
|
+
ensembleConfidence?: number;
|
|
14
|
+
findingCount?: number;
|
|
15
|
+
}
|
|
16
|
+
/** Check the audit's report.json for anti-bot patterns. Returns the reason or null. */
|
|
17
|
+
export declare function detectBlock(report: unknown): string | null;
|
|
18
|
+
export declare function reasonFor(s: BlockSignals): string | null;
|
|
19
|
+
//# sourceMappingURL=anti-bot.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"anti-bot.d.ts","sourceRoot":"","sources":["../../src/jobs/anti-bot.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAsBH,MAAM,WAAW,YAAY;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,uFAAuF;AACvF,wBAAgB,WAAW,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAc1D;AAED,wBAAgB,SAAS,CAAC,CAAC,EAAE,YAAY,GAAG,MAAM,GAAG,IAAI,CAmBxD"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Anti-bot / blocked-page detection. Pure pattern match against an audit's
|
|
3
|
+
* report.json — we propagate the existing audit signals rather than re-running
|
|
4
|
+
* inference.
|
|
5
|
+
*
|
|
6
|
+
* Returns a reason string when blocked (so the job can carry it through to
|
|
7
|
+
* the result envelope), else null.
|
|
8
|
+
*/
|
|
9
|
+
const TITLE_PATTERNS = [
|
|
10
|
+
/just a moment\.{3}/i,
|
|
11
|
+
/^attention required/i,
|
|
12
|
+
/access denied/i,
|
|
13
|
+
/verify you are human/i,
|
|
14
|
+
/enable javascript and cookies/i,
|
|
15
|
+
/one more step/i,
|
|
16
|
+
/please complete the security check/i,
|
|
17
|
+
/^cloudflare/i,
|
|
18
|
+
/challenge[- ]page/i,
|
|
19
|
+
];
|
|
20
|
+
const INTENT_PATTERNS = [
|
|
21
|
+
/cloudflare challenge/i,
|
|
22
|
+
/anti.?bot/i,
|
|
23
|
+
/captcha/i,
|
|
24
|
+
/verify (the )?(human|user|browser)/i,
|
|
25
|
+
/access (denied|restricted|blocked)/i,
|
|
26
|
+
];
|
|
27
|
+
/** Check the audit's report.json for anti-bot patterns. Returns the reason or null. */
|
|
28
|
+
export function detectBlock(report) {
|
|
29
|
+
const r = report;
|
|
30
|
+
const page = r.pages?.[0];
|
|
31
|
+
if (!page)
|
|
32
|
+
return null;
|
|
33
|
+
const v2cls = page.auditResultV2?.classification;
|
|
34
|
+
const cls = v2cls ?? page.classification ?? {};
|
|
35
|
+
const signals = {
|
|
36
|
+
title: page.title,
|
|
37
|
+
intent: cls.intent,
|
|
38
|
+
type: cls.type,
|
|
39
|
+
ensembleConfidence: cls.ensembleConfidence,
|
|
40
|
+
findingCount: page.findings?.length ?? 0,
|
|
41
|
+
};
|
|
42
|
+
return reasonFor(signals);
|
|
43
|
+
}
|
|
44
|
+
export function reasonFor(s) {
|
|
45
|
+
const title = (s.title ?? '').trim();
|
|
46
|
+
const intent = (s.intent ?? '').trim();
|
|
47
|
+
if (TITLE_PATTERNS.some(re => re.test(title))) {
|
|
48
|
+
return `blocked: page title looks like an anti-bot challenge ("${title.slice(0, 80)}")`;
|
|
49
|
+
}
|
|
50
|
+
if (INTENT_PATTERNS.some(re => re.test(intent))) {
|
|
51
|
+
return `blocked: classification intent indicates a challenge page ("${intent.slice(0, 80)}")`;
|
|
52
|
+
}
|
|
53
|
+
// Last-resort heuristic: zero findings + very low ensemble confidence + unknown
|
|
54
|
+
// page-type is overwhelmingly an anti-bot or empty page. Leaving it in the
|
|
55
|
+
// leaderboard pollutes rankings.
|
|
56
|
+
if ((s.findingCount ?? 0) === 0
|
|
57
|
+
&& typeof s.ensembleConfidence === 'number'
|
|
58
|
+
&& s.ensembleConfidence < 0.35
|
|
59
|
+
&& s.type === 'unknown') {
|
|
60
|
+
return 'blocked: zero findings, low classifier confidence, unknown type — likely empty/blocked';
|
|
61
|
+
}
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=anti-bot.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"anti-bot.js","sourceRoot":"","sources":["../../src/jobs/anti-bot.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,cAAc,GAAG;IACrB,qBAAqB;IACrB,sBAAsB;IACtB,gBAAgB;IAChB,uBAAuB;IACvB,gCAAgC;IAChC,gBAAgB;IAChB,qCAAqC;IACrC,cAAc;IACd,oBAAoB;CACrB,CAAA;AAED,MAAM,eAAe,GAAG;IACtB,uBAAuB;IACvB,YAAY;IACZ,UAAU;IACV,qCAAqC;IACrC,qCAAqC;CACtC,CAAA;AAUD,uFAAuF;AACvF,MAAM,UAAU,WAAW,CAAC,MAAe;IACzC,MAAM,CAAC,GAAG,MAA6P,CAAA;IACvQ,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;IACzB,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,cAAc,CAAA;IAChD,MAAM,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,cAAc,IAAI,EAAE,CAAA;IAC9C,MAAM,OAAO,GAAiB;QAC5B,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;QAC1C,YAAY,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC;KACzC,CAAA;IACD,OAAO,SAAS,CAAC,OAAO,CAAC,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,CAAe;IACvC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IACpC,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IACtC,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAC9C,OAAO,0DAA0D,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAA;IACzF,CAAC;IACD,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QAChD,OAAO,+DAA+D,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAA;IAC/F,CAAC;IACD,gFAAgF;IAChF,2EAA2E;IAC3E,iCAAiC;IACjC,IAAI,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,KAAK,CAAC;WAC1B,OAAO,CAAC,CAAC,kBAAkB,KAAK,QAAQ;WACxC,CAAC,CAAC,kBAAkB,GAAG,IAAI;WAC3B,CAAC,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,wFAAwF,CAAA;IACjG,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pre-flight cost estimate — refuses to silently spend $1k.
|
|
3
|
+
*
|
|
4
|
+
* The numbers here are deliberately conservative and based on observed
|
|
5
|
+
* production telemetry for `claude-code` provider with sonnet at the time of
|
|
6
|
+
* writing. If the assumptions drift, only one constant changes.
|
|
7
|
+
*/
|
|
8
|
+
import type { CostEstimate, JobSpec } from './types.js';
|
|
9
|
+
/** Average USD per single-page audit. Tuned for sonnet via claude-code provider, 1 audit pass. */
|
|
10
|
+
export declare const DEFAULT_PER_AUDIT_USD = 0.4;
|
|
11
|
+
export declare function estimateCost(spec: JobSpec, targetCount: number, perAuditUSD?: number): CostEstimate;
|
|
12
|
+
//# sourceMappingURL=cost-estimate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cost-estimate.d.ts","sourceRoot":"","sources":["../../src/jobs/cost-estimate.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AAEvD,kGAAkG;AAClG,eAAO,MAAM,qBAAqB,MAAM,CAAA;AAExC,wBAAgB,YAAY,CAAC,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,SAAwB,GAAG,YAAY,CAUlH"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pre-flight cost estimate — refuses to silently spend $1k.
|
|
3
|
+
*
|
|
4
|
+
* The numbers here are deliberately conservative and based on observed
|
|
5
|
+
* production telemetry for `claude-code` provider with sonnet at the time of
|
|
6
|
+
* writing. If the assumptions drift, only one constant changes.
|
|
7
|
+
*/
|
|
8
|
+
/** Average USD per single-page audit. Tuned for sonnet via claude-code provider, 1 audit pass. */
|
|
9
|
+
export const DEFAULT_PER_AUDIT_USD = 0.4;
|
|
10
|
+
export function estimateCost(spec, targetCount, perAuditUSD = DEFAULT_PER_AUDIT_USD) {
|
|
11
|
+
const pages = Math.max(spec.audit?.pages ?? 1, 1);
|
|
12
|
+
const estimatedTotalUSD = targetCount * pages * perAuditUSD;
|
|
13
|
+
const exceedsCap = typeof spec.maxCostUSD === 'number' && estimatedTotalUSD > spec.maxCostUSD;
|
|
14
|
+
return {
|
|
15
|
+
targetCount,
|
|
16
|
+
perAuditUSD: perAuditUSD * pages,
|
|
17
|
+
estimatedTotalUSD,
|
|
18
|
+
exceedsCap,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=cost-estimate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cost-estimate.js","sourceRoot":"","sources":["../../src/jobs/cost-estimate.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,kGAAkG;AAClG,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAExC,MAAM,UAAU,YAAY,CAAC,IAAa,EAAE,WAAmB,EAAE,WAAW,GAAG,qBAAqB;IAClG,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;IACjD,MAAM,iBAAiB,GAAG,WAAW,GAAG,KAAK,GAAG,WAAW,CAAA;IAC3D,MAAM,UAAU,GAAG,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,IAAI,iBAAiB,GAAG,IAAI,CAAC,UAAU,CAAA;IAC7F,OAAO;QACL,WAAW;QACX,WAAW,EAAE,WAAW,GAAG,KAAK;QAChC,iBAAiB;QACjB,UAAU;KACX,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adaptive cost estimate from historical jobs. The default flat
|
|
3
|
+
* `DEFAULT_PER_AUDIT_USD` is still a fine starting point for a fresh user,
|
|
4
|
+
* but once 3+ jobs have completed we can do better: averaging the actual
|
|
5
|
+
* per-target cost across recent jobs is closer to ground truth, especially
|
|
6
|
+
* once ethics / first-principles modes start firing differently per target.
|
|
7
|
+
*
|
|
8
|
+
* Pure function of `~/.bad/jobs/` records — no telemetry endpoint required.
|
|
9
|
+
*/
|
|
10
|
+
export interface AdaptiveCostStats {
|
|
11
|
+
perAuditUSD: number;
|
|
12
|
+
source: 'history' | 'default';
|
|
13
|
+
/** Number of historical job records the estimate was averaged over. */
|
|
14
|
+
jobsObserved: number;
|
|
15
|
+
/** Number of audited targets the estimate was averaged over. */
|
|
16
|
+
targetsObserved: number;
|
|
17
|
+
}
|
|
18
|
+
export declare function computePerAuditFromHistory(dir?: string): AdaptiveCostStats;
|
|
19
|
+
//# sourceMappingURL=cost-history.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cost-history.d.ts","sourceRoot":"","sources":["../../src/jobs/cost-history.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AASH,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,SAAS,GAAG,SAAS,CAAA;IAC7B,uEAAuE;IACvE,YAAY,EAAE,MAAM,CAAA;IACpB,gEAAgE;IAChE,eAAe,EAAE,MAAM,CAAA;CACxB;AAED,wBAAgB,0BAA0B,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,iBAAiB,CAyB1E"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adaptive cost estimate from historical jobs. The default flat
|
|
3
|
+
* `DEFAULT_PER_AUDIT_USD` is still a fine starting point for a fresh user,
|
|
4
|
+
* but once 3+ jobs have completed we can do better: averaging the actual
|
|
5
|
+
* per-target cost across recent jobs is closer to ground truth, especially
|
|
6
|
+
* once ethics / first-principles modes start firing differently per target.
|
|
7
|
+
*
|
|
8
|
+
* Pure function of `~/.bad/jobs/` records — no telemetry endpoint required.
|
|
9
|
+
*/
|
|
10
|
+
import { listJobs, loadJob } from './store.js';
|
|
11
|
+
import { DEFAULT_PER_AUDIT_USD } from './cost-estimate.js';
|
|
12
|
+
/** Min number of completed jobs before we trust history over the static default. */
|
|
13
|
+
const MIN_HISTORY = 3;
|
|
14
|
+
export function computePerAuditFromHistory(dir) {
|
|
15
|
+
const entries = listJobs(dir);
|
|
16
|
+
// Only count completed/partial jobs — failed ones have skewed cost.
|
|
17
|
+
const usable = entries.filter((e) => e.status === 'completed' || e.status === 'partial').slice(0, 20);
|
|
18
|
+
if (usable.length < MIN_HISTORY) {
|
|
19
|
+
return { perAuditUSD: DEFAULT_PER_AUDIT_USD, source: 'default', jobsObserved: usable.length, targetsObserved: 0 };
|
|
20
|
+
}
|
|
21
|
+
let totalCost = 0;
|
|
22
|
+
let totalTargets = 0;
|
|
23
|
+
for (const entry of usable) {
|
|
24
|
+
const job = loadJob(entry.jobId, dir);
|
|
25
|
+
if (!job)
|
|
26
|
+
continue;
|
|
27
|
+
const okCount = job.results.filter(r => r.status === 'ok' && typeof r.costUSD === 'number').length;
|
|
28
|
+
if (okCount === 0)
|
|
29
|
+
continue;
|
|
30
|
+
totalCost += job.totalCostUSD;
|
|
31
|
+
totalTargets += okCount;
|
|
32
|
+
}
|
|
33
|
+
if (totalTargets === 0) {
|
|
34
|
+
return { perAuditUSD: DEFAULT_PER_AUDIT_USD, source: 'default', jobsObserved: usable.length, targetsObserved: 0 };
|
|
35
|
+
}
|
|
36
|
+
const perAudit = totalCost / totalTargets;
|
|
37
|
+
// Floor at half the static default to prevent runaway optimism on a stretch
|
|
38
|
+
// of zero-cost jobs (which can happen with the claude-code provider).
|
|
39
|
+
const floored = Math.max(perAudit, DEFAULT_PER_AUDIT_USD * 0.5);
|
|
40
|
+
return { perAuditUSD: floored, source: 'history', jobsObserved: usable.length, targetsObserved: totalTargets };
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=cost-history.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cost-history.js","sourceRoot":"","sources":["../../src/jobs/cost-history.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AAC9C,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAE1D,oFAAoF;AACpF,MAAM,WAAW,GAAG,CAAC,CAAA;AAWrB,MAAM,UAAU,0BAA0B,CAAC,GAAY;IACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;IAC7B,oEAAoE;IACpE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IACpH,IAAI,MAAM,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;QAChC,OAAO,EAAE,WAAW,EAAE,qBAAqB,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC,EAAE,CAAA;IACnH,CAAC;IACD,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;QACrC,IAAI,CAAC,GAAG;YAAE,SAAQ;QAClB,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;QAClG,IAAI,OAAO,KAAK,CAAC;YAAE,SAAQ;QAC3B,SAAS,IAAI,GAAG,CAAC,YAAY,CAAA;QAC7B,YAAY,IAAI,OAAO,CAAA;IACzB,CAAC;IACD,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,WAAW,EAAE,qBAAqB,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC,EAAE,CAAA;IACnH,CAAC;IACD,MAAM,QAAQ,GAAG,SAAS,GAAG,YAAY,CAAA;IACzC,4EAA4E;IAC5E,sEAAsE;IACtE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,qBAAqB,GAAG,GAAG,CAAC,CAAA;IAC/D,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,EAAE,YAAY,EAAE,CAAA;AAChH,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jobs — declarative comparative-audit jobs.
|
|
3
|
+
*
|
|
4
|
+
* Public surface:
|
|
5
|
+
* createJob(spec) → Job (queued, persisted, ready to run)
|
|
6
|
+
* runJob(job, opts) → Job (executes the fan-out, returns final state)
|
|
7
|
+
* loadJob / listJobs / saveJob — store accessors
|
|
8
|
+
* estimateCost — pre-flight cost guard
|
|
9
|
+
*/
|
|
10
|
+
export type { Job, JobSpec, JobStatus, JobKind, JobTarget, JobResultEntry, JobResultStatus, DiscoverSpec, DiscoverSource, AuditOptions, CostEstimate, } from './types.js';
|
|
11
|
+
export { newJobId, saveJob, loadJob, listJobs, appendIndexEntry, jobsDir, jobPath, updateJobStatus } from './store.js';
|
|
12
|
+
export type { JobIndexEntry } from './store.js';
|
|
13
|
+
export { estimateCost, DEFAULT_PER_AUDIT_USD } from './cost-estimate.js';
|
|
14
|
+
export { runJob } from './queue.js';
|
|
15
|
+
export type { AuditFn, RunJobOptions } from './queue.js';
|
|
16
|
+
export { withRetry, isRetryableDefault, DEFAULT_RETRY_POLICY } from './retry.js';
|
|
17
|
+
export type { RetryPolicy } from './retry.js';
|
|
18
|
+
export { detectBlock, reasonFor } from './anti-bot.js';
|
|
19
|
+
export type { BlockSignals } from './anti-bot.js';
|
|
20
|
+
export { computePerAuditFromHistory } from './cost-history.js';
|
|
21
|
+
export type { AdaptiveCostStats } from './cost-history.js';
|
|
22
|
+
export { orchestrateJob, needsIntervention } from './orchestrator.js';
|
|
23
|
+
export type { OrchestrateJobOptions } from './orchestrator.js';
|
|
24
|
+
import type { Job, JobSpec, JobTarget } from './types.js';
|
|
25
|
+
/**
|
|
26
|
+
* Mint a fresh `queued` job from a spec. Targets are seeded from the spec
|
|
27
|
+
* (snapshot expansion, if any, must be done by the caller via the
|
|
28
|
+
* `discover` module before runJob).
|
|
29
|
+
*/
|
|
30
|
+
export declare function createJob(spec: JobSpec, targets: JobTarget[], dir?: string): Job;
|
|
31
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/jobs/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,YAAY,EACV,GAAG,EACH,OAAO,EACP,SAAS,EACT,OAAO,EACP,SAAS,EACT,cAAc,EACd,eAAe,EACf,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,YAAY,GACb,MAAM,YAAY,CAAA;AAEnB,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AACtH,YAAY,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAC/C,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AACxE,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AACnC,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AACxD,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AAChF,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAC7C,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AACtD,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AACjD,OAAO,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAA;AAC9D,YAAY,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AACrE,YAAY,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAA;AAE9D,OAAO,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAGzD;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,GAAG,CAahF"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jobs — declarative comparative-audit jobs.
|
|
3
|
+
*
|
|
4
|
+
* Public surface:
|
|
5
|
+
* createJob(spec) → Job (queued, persisted, ready to run)
|
|
6
|
+
* runJob(job, opts) → Job (executes the fan-out, returns final state)
|
|
7
|
+
* loadJob / listJobs / saveJob — store accessors
|
|
8
|
+
* estimateCost — pre-flight cost guard
|
|
9
|
+
*/
|
|
10
|
+
export { newJobId, saveJob, loadJob, listJobs, appendIndexEntry, jobsDir, jobPath, updateJobStatus } from './store.js';
|
|
11
|
+
export { estimateCost, DEFAULT_PER_AUDIT_USD } from './cost-estimate.js';
|
|
12
|
+
export { runJob } from './queue.js';
|
|
13
|
+
export { withRetry, isRetryableDefault, DEFAULT_RETRY_POLICY } from './retry.js';
|
|
14
|
+
export { detectBlock, reasonFor } from './anti-bot.js';
|
|
15
|
+
export { computePerAuditFromHistory } from './cost-history.js';
|
|
16
|
+
export { orchestrateJob, needsIntervention } from './orchestrator.js';
|
|
17
|
+
import { newJobId, saveJob, appendIndexEntry } from './store.js';
|
|
18
|
+
/**
|
|
19
|
+
* Mint a fresh `queued` job from a spec. Targets are seeded from the spec
|
|
20
|
+
* (snapshot expansion, if any, must be done by the caller via the
|
|
21
|
+
* `discover` module before runJob).
|
|
22
|
+
*/
|
|
23
|
+
export function createJob(spec, targets, dir) {
|
|
24
|
+
const job = {
|
|
25
|
+
jobId: newJobId(),
|
|
26
|
+
spec,
|
|
27
|
+
status: 'queued',
|
|
28
|
+
createdAt: new Date().toISOString(),
|
|
29
|
+
targets,
|
|
30
|
+
results: [],
|
|
31
|
+
totalCostUSD: 0,
|
|
32
|
+
};
|
|
33
|
+
saveJob(job, dir);
|
|
34
|
+
appendIndexEntry(job, dir);
|
|
35
|
+
return job;
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/jobs/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAgBH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAEtH,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AACxE,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAEnC,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AAEhF,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AAEtD,OAAO,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAA;AAE9D,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AAIrE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAEhE;;;;GAIG;AACH,MAAM,UAAU,SAAS,CAAC,IAAa,EAAE,OAAoB,EAAE,GAAY;IACzE,MAAM,GAAG,GAAQ;QACf,KAAK,EAAE,QAAQ,EAAE;QACjB,IAAI;QACJ,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,OAAO;QACP,OAAO,EAAE,EAAE;QACX,YAAY,EAAE,CAAC;KAChB,CAAA;IACD,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;IACjB,gBAAgB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;IAC1B,OAAO,GAAG,CAAA;AACZ,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Agentic orchestrator — wraps `runJob` with a control loop that handles
|
|
3
|
+
* strategic decisions (re-sample broken wayback snapshots, retry vs skip,
|
|
4
|
+
* widen the window when too many fail, conclude) via LLM tool calls.
|
|
5
|
+
*
|
|
6
|
+
* Design: protocols are deterministic (retry/backoff lives in `retry.ts`,
|
|
7
|
+
* anti-bot detection lives in `anti-bot.ts`). The orchestrator only steps
|
|
8
|
+
* in for *judgment* calls where a hand-tuned heuristic would break the
|
|
9
|
+
* moment the world changes. Cost ceiling: hard-capped at spec.maxCostUSD,
|
|
10
|
+
* and each control-loop iteration is a single LLM call (~1.5s, ~$0.005).
|
|
11
|
+
*
|
|
12
|
+
* Tool surface (LLM-callable):
|
|
13
|
+
* getJobState() — what's done, what failed, current cost
|
|
14
|
+
* resampleWayback(...) — replace targets for a URL with a new sample
|
|
15
|
+
* retryTarget(...) — re-attempt a single failed/skipped target
|
|
16
|
+
* markSkipped(...) — terminal skip with reason
|
|
17
|
+
* concludeJob() — exit the loop
|
|
18
|
+
*
|
|
19
|
+
* The orchestrator runs the initial fan-out via `runJob` first (so the
|
|
20
|
+
* deterministic happy path doesn't pay LLM tax), then enters the loop only
|
|
21
|
+
* if the result needs intervention.
|
|
22
|
+
*/
|
|
23
|
+
import { Brain } from '../brain/index.js';
|
|
24
|
+
import { type AuditFn } from './queue.js';
|
|
25
|
+
import type { Job } from './types.js';
|
|
26
|
+
export interface OrchestrateJobOptions {
|
|
27
|
+
auditFn: AuditFn;
|
|
28
|
+
/** Persistence dir override (tests). */
|
|
29
|
+
dir?: string;
|
|
30
|
+
/** Cap on agent control-loop iterations. Default 8. */
|
|
31
|
+
maxIterations?: number;
|
|
32
|
+
/** Override the LLM provider (tests). Defaults to the default Brain. */
|
|
33
|
+
brain?: Brain;
|
|
34
|
+
/** When true, log every tool call. */
|
|
35
|
+
verbose?: boolean;
|
|
36
|
+
}
|
|
37
|
+
export declare function orchestrateJob(job: Job, opts: OrchestrateJobOptions): Promise<Job>;
|
|
38
|
+
/** Cheap deterministic check: did the fan-out leave anything worth fixing? */
|
|
39
|
+
declare function needsIntervention(job: Job): boolean;
|
|
40
|
+
/** Re-export for tests / callers that want to inspect what would have run. */
|
|
41
|
+
export { needsIntervention };
|
|
42
|
+
//# sourceMappingURL=orchestrator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/jobs/orchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAGH,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAA;AAEzC,OAAO,EAAU,KAAK,OAAO,EAAE,MAAM,YAAY,CAAA;AAGjD,OAAO,KAAK,EAAE,GAAG,EAA6B,MAAM,YAAY,CAAA;AAEhE,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAA;IAChB,wCAAwC;IACxC,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,uDAAuD;IACvD,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,wEAAwE;IACxE,KAAK,CAAC,EAAE,KAAK,CAAA;IACb,sCAAsC;IACtC,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB;AAkCD,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,qBAAqB,GAAG,OAAO,CAAC,GAAG,CAAC,CAmDxF;AAED,8EAA8E;AAC9E,iBAAS,iBAAiB,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAa5C;AA+JD,8EAA8E;AAC9E,OAAO,EAAE,iBAAiB,EAAE,CAAA"}
|