@q32/signal-scanner 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/dist/dynamic.d.ts +43 -0
  2. package/dist/dynamic.d.ts.map +1 -0
  3. package/{src/dynamic.ts → dist/dynamic.js} +133 -156
  4. package/dist/dynamic.js.map +1 -0
  5. package/dist/feeds.d.ts +66 -0
  6. package/dist/feeds.d.ts.map +1 -0
  7. package/dist/feeds.js +259 -0
  8. package/dist/feeds.js.map +1 -0
  9. package/dist/index.d.ts +110 -0
  10. package/dist/index.d.ts.map +1 -0
  11. package/dist/index.js +1251 -0
  12. package/dist/index.js.map +1 -0
  13. package/dist/intel.d.ts +72 -0
  14. package/dist/intel.d.ts.map +1 -0
  15. package/dist/intel.js +480 -0
  16. package/dist/intel.js.map +1 -0
  17. package/dist/node-tls.d.ts +8 -0
  18. package/dist/node-tls.d.ts.map +1 -0
  19. package/dist/node-tls.js +48 -0
  20. package/dist/node-tls.js.map +1 -0
  21. package/dist/render.d.ts +26 -0
  22. package/dist/render.d.ts.map +1 -0
  23. package/dist/render.js +248 -0
  24. package/dist/render.js.map +1 -0
  25. package/dist/rules/packs/binary.d.ts +4 -0
  26. package/dist/rules/packs/binary.d.ts.map +1 -0
  27. package/dist/rules/packs/binary.js +101 -0
  28. package/dist/rules/packs/binary.js.map +1 -0
  29. package/dist/rules/packs/css.d.ts +3 -0
  30. package/dist/rules/packs/css.d.ts.map +1 -0
  31. package/dist/rules/packs/css.js +43 -0
  32. package/dist/rules/packs/css.js.map +1 -0
  33. package/dist/rules/packs/decoders.d.ts +3 -0
  34. package/dist/rules/packs/decoders.d.ts.map +1 -0
  35. package/dist/rules/packs/decoders.js +46 -0
  36. package/dist/rules/packs/decoders.js.map +1 -0
  37. package/dist/rules/packs/html.d.ts +4 -0
  38. package/dist/rules/packs/html.d.ts.map +1 -0
  39. package/dist/rules/packs/html.js +227 -0
  40. package/dist/rules/packs/html.js.map +1 -0
  41. package/dist/rules/packs/index.d.ts +24 -0
  42. package/dist/rules/packs/index.d.ts.map +1 -0
  43. package/dist/rules/packs/index.js +75 -0
  44. package/dist/rules/packs/index.js.map +1 -0
  45. package/dist/rules/packs/script-risk.d.ts +4 -0
  46. package/dist/rules/packs/script-risk.d.ts.map +1 -0
  47. package/dist/rules/packs/script-risk.js +231 -0
  48. package/dist/rules/packs/script-risk.js.map +1 -0
  49. package/dist/rules/packs/source-code.d.ts +3 -0
  50. package/dist/rules/packs/source-code.d.ts.map +1 -0
  51. package/dist/rules/packs/source-code.js +179 -0
  52. package/dist/rules/packs/source-code.js.map +1 -0
  53. package/dist/rules/packs/urls.d.ts +3 -0
  54. package/dist/rules/packs/urls.d.ts.map +1 -0
  55. package/dist/rules/packs/urls.js +123 -0
  56. package/dist/rules/packs/urls.js.map +1 -0
  57. package/dist/rules/types.d.ts +34 -0
  58. package/dist/rules/types.d.ts.map +1 -0
  59. package/dist/rules/types.js +2 -0
  60. package/dist/rules/types.js.map +1 -0
  61. package/package.json +18 -14
  62. package/src/feeds.ts +0 -334
  63. package/src/index.ts +0 -1366
  64. package/src/intel.ts +0 -561
  65. package/src/node-tls.ts +0 -55
  66. package/src/render.ts +0 -233
  67. package/src/rules/packs/binary.ts +0 -103
  68. package/src/rules/packs/css.ts +0 -44
  69. package/src/rules/packs/decoders.ts +0 -47
  70. package/src/rules/packs/html.ts +0 -255
  71. package/src/rules/packs/index.ts +0 -76
  72. package/src/rules/packs/script-risk.ts +0 -236
  73. package/src/rules/packs/source-code.ts +0 -180
  74. package/src/rules/packs/urls.ts +0 -138
  75. package/src/rules/types.ts +0 -56
@@ -0,0 +1,43 @@
1
+ import { type Finding } from "./index.js";
2
+ export interface NetworkAttempt {
3
+ kind: "fetch" | "xhr" | "beacon" | "websocket" | "script" | "image" | "form";
4
+ url: string;
5
+ }
6
+ export interface BehaviorReport {
7
+ redirects: string[];
8
+ network: NetworkAttempt[];
9
+ writes: string[];
10
+ evals: string[];
11
+ decoded: string[];
12
+ cookies: string[];
13
+ errors: string[];
14
+ }
15
+ export interface DynamicAnalysisOptions {
16
+ /** Base/page URL, used to resolve relative targets and classify off-origin. */
17
+ url?: string;
18
+ }
19
+ /** The caller supplies one of these: "run these scripts in an isolate, give me what they attempted." */
20
+ export type IsolatedEvaluator = (scripts: string[], options: DynamicAnalysisOptions) => BehaviorReport | Promise<BehaviorReport>;
21
+ export declare const RECORDER_SOURCE: string;
22
+ /** Extract inline <script> bodies (no src) from HTML. */
23
+ export declare function extractInlineScripts(html: string): string[];
24
+ export declare function extractScriptSources(html: string): string[];
25
+ /**
26
+ * In-process default evaluator. Runs the recorder in THIS isolate (no boundary).
27
+ * Use analyzeDynamicWith with a caller-supplied evaluator when isolation matters.
28
+ */
29
+ export declare function runInstrumented(scripts: string[], options?: DynamicAnalysisOptions): BehaviorReport;
30
+ /** Full in-process pass: extract inline scripts, record behavior, turn it into findings. */
31
+ export declare function analyzeDynamic(html: string, options?: DynamicAnalysisOptions): {
32
+ report: BehaviorReport;
33
+ findings: Finding[];
34
+ };
35
+ /** Generic seam: the caller supplies how scripts are evaluated (in whatever isolate it has). */
36
+ export declare function analyzeDynamicWith(html: string, options: DynamicAnalysisOptions, evaluate: IsolatedEvaluator): Promise<{
37
+ report: BehaviorReport;
38
+ findings: Finding[];
39
+ }>;
40
+ /** Map recorded behavior to scanner findings, re-scanning injected markup and decoded/eval'd code. */
41
+ export declare function behaviorFindings(report: BehaviorReport, baseUrl?: string): Finding[];
42
+ export declare function discoveredUrlsFromBehavior(report: BehaviorReport, baseUrl?: string): string[];
43
+ //# sourceMappingURL=dynamic.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"dynamic.d.ts","sourceRoot":"","sources":["../src/dynamic.ts"],"names":[],"mappings":"AAmBA,OAAO,EAA4E,KAAK,OAAO,EAAkC,MAAM,YAAY,CAAC;AAGpJ,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,OAAO,GAAG,KAAK,GAAG,QAAQ,GAAG,WAAW,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,CAAC;IAC7E,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,+EAA+E;IAC/E,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,wGAAwG;AACxG,MAAM,MAAM,iBAAiB,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,sBAAsB,KAAK,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;AAUjI,eAAO,MAAM,eAAe,QA6D3B,CAAC;AAWF,yDAAyD;AACzD,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAU3D;AAID,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAM3D;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,sBAA2B,GAAG,cAAc,CAMvG;AAED,4FAA4F;AAC5F,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,sBAA2B,GAAG;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,QAAQ,EAAE,OAAO,EAAE,CAAA;CAAE,CAGlI;AAED,gGAAgG;AAChG,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,sBAAsB,EAC/B,QAAQ,EAAE,iBAAiB,GAC1B,OAAO,CAAC;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,QAAQ,EAAE,OAAO,EAAE,CAAA;CAAE,CAAC,CAI1D;AASD,sGAAsG;AACtG,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,EAAE,CA2CpF;AAOD,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAW7F"}
@@ -16,42 +16,15 @@
16
16
  // for when no isolate boundary is needed.
17
17
  // - analyzeDynamicWith(html, opts, evaluate): the generic seam — the caller
18
18
  // passes an `evaluate` that produces a BehaviorReport however it likes.
19
-
20
- import { assessRedirect, createScanner, isAdOrAnalyticsHost, registrableDomainFor, type Finding, type Severity, type Confidence } from "./index";
21
- import type { RuleScoreModel } from "./rules/types";
22
-
23
- export interface NetworkAttempt {
24
- kind: "fetch" | "xhr" | "beacon" | "websocket" | "script" | "image" | "form";
25
- url: string;
26
- }
27
-
28
- export interface BehaviorReport {
29
- redirects: string[];
30
- network: NetworkAttempt[];
31
- writes: string[];
32
- evals: string[];
33
- decoded: string[];
34
- cookies: string[];
35
- errors: string[];
36
- }
37
-
38
- export interface DynamicAnalysisOptions {
39
- /** Base/page URL, used to resolve relative targets and classify off-origin. */
40
- url?: string;
41
- }
42
-
43
- /** The caller supplies one of these: "run these scripts in an isolate, give me what they attempted." */
44
- export type IsolatedEvaluator = (scripts: string[], options: DynamicAnalysisOptions) => BehaviorReport | Promise<BehaviorReport>;
45
-
46
- const EMPTY_REPORT: BehaviorReport = { redirects: [], network: [], writes: [], evals: [], decoded: [], cookies: [], errors: [] };
47
-
19
+ import { assessRedirect, createScanner, isAdOrAnalyticsHost, registrableDomainFor } from "./index.js";
20
+ const EMPTY_REPORT = { redirects: [], network: [], writes: [], evals: [], decoded: [], cookies: [], errors: [] };
48
21
  // Self-contained recorder. Evaluating this source defines `recordBehavior(scripts, url)`
49
22
  // which returns a BehaviorReport. It references only standard globals (URL, atob,
50
23
  // btoa, Proxy) available in any JS isolate — no imports, no transport, no
51
24
  // assumptions about how it is hosted. Inline scripts run via `new Function` with
52
25
  // the dangerous globals shadowed by recorder stubs (sloppy mode so `eval` /
53
26
  // `Function` can be shadowed as parameters).
54
- export const RECORDER_SOURCE = String.raw`
27
+ export const RECORDER_SOURCE = String.raw `
55
28
  function recordBehavior(scripts, url) {
56
29
  var report = { redirects: [], network: [], writes: [], evals: [], decoded: [], cookies: [], errors: [] };
57
30
  var resolve = function (v) { var raw = String(v == null ? "" : v); try { return url ? new URL(raw, url).toString() : raw; } catch (e) { return raw; } };
@@ -113,161 +86,165 @@ function recordBehavior(scripts, url) {
113
86
  return report;
114
87
  }
115
88
  `;
116
-
117
- let compiledRecorder: ((scripts: string[], url?: string) => BehaviorReport) | null = null;
118
- function inProcessRecorder(): (scripts: string[], url?: string) => BehaviorReport {
119
- if (!compiledRecorder) {
120
- // eslint-disable-next-line no-new-func
121
- compiledRecorder = new Function(`${RECORDER_SOURCE}\nreturn recordBehavior;`)() as (scripts: string[], url?: string) => BehaviorReport;
122
- }
123
- return compiledRecorder;
89
+ let compiledRecorder = null;
90
+ function inProcessRecorder() {
91
+ if (!compiledRecorder) {
92
+ // eslint-disable-next-line no-new-func
93
+ compiledRecorder = new Function(`${RECORDER_SOURCE}\nreturn recordBehavior;`)();
94
+ }
95
+ return compiledRecorder;
124
96
  }
125
-
126
97
  /** Extract inline <script> bodies (no src) from HTML. */
127
- export function extractInlineScripts(html: string): string[] {
128
- const scripts: string[] = [];
129
- for (const match of html.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script>/gi)) {
130
- const attrs = match[1] ?? "";
131
- if (/\bsrc\s*=/i.test(attrs)) continue; // external scripts are fetched + scanned separately
132
- if (/\btype\s*=\s*["']?(?:application\/json|application\/ld\+json|text\/template)/i.test(attrs)) continue;
133
- const body = match[2]?.trim();
134
- if (body) scripts.push(body);
135
- }
136
- return scripts;
98
+ export function extractInlineScripts(html) {
99
+ const scripts = [];
100
+ for (const match of html.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script>/gi)) {
101
+ const attrs = match[1] ?? "";
102
+ if (/\bsrc\s*=/i.test(attrs))
103
+ continue; // external scripts are fetched + scanned separately
104
+ if (/\btype\s*=\s*["']?(?:application\/json|application\/ld\+json|text\/template)/i.test(attrs))
105
+ continue;
106
+ const body = match[2]?.trim();
107
+ if (body)
108
+ scripts.push(body);
109
+ }
110
+ return scripts;
137
111
  }
138
-
139
112
  // Source URLs of external scripts the page loads (<script src=...>). A renderer
140
113
  // fetches and runs these against the DOM, where externally-injected forms appear.
141
- export function extractScriptSources(html: string): string[] {
142
- const sources: string[] = [];
143
- for (const match of html.matchAll(/<script\b[^>]*\bsrc\s*=\s*["']([^"']+)["'][^>]*>/gi)) {
144
- if (match[1]) sources.push(match[1]);
145
- }
146
- return [...new Set(sources)];
114
+ export function extractScriptSources(html) {
115
+ const sources = [];
116
+ for (const match of html.matchAll(/<script\b[^>]*\bsrc\s*=\s*["']([^"']+)["'][^>]*>/gi)) {
117
+ if (match[1])
118
+ sources.push(match[1]);
119
+ }
120
+ return [...new Set(sources)];
147
121
  }
148
-
149
122
  /**
150
123
  * In-process default evaluator. Runs the recorder in THIS isolate (no boundary).
151
124
  * Use analyzeDynamicWith with a caller-supplied evaluator when isolation matters.
152
125
  */
153
- export function runInstrumented(scripts: string[], options: DynamicAnalysisOptions = {}): BehaviorReport {
154
- try {
155
- return inProcessRecorder()(scripts, options.url);
156
- } catch (error) {
157
- return { ...EMPTY_REPORT, errors: [error instanceof Error ? error.message : "recorder failed"] };
158
- }
126
+ export function runInstrumented(scripts, options = {}) {
127
+ try {
128
+ return inProcessRecorder()(scripts, options.url);
129
+ }
130
+ catch (error) {
131
+ return { ...EMPTY_REPORT, errors: [error instanceof Error ? error.message : "recorder failed"] };
132
+ }
159
133
  }
160
-
161
134
  /** Full in-process pass: extract inline scripts, record behavior, turn it into findings. */
162
- export function analyzeDynamic(html: string, options: DynamicAnalysisOptions = {}): { report: BehaviorReport; findings: Finding[] } {
163
- const report = runInstrumented(extractInlineScripts(html), options);
164
- return { report, findings: behaviorFindings(report, options.url) };
135
+ export function analyzeDynamic(html, options = {}) {
136
+ const report = runInstrumented(extractInlineScripts(html), options);
137
+ return { report, findings: behaviorFindings(report, options.url) };
165
138
  }
166
-
167
139
  /** Generic seam: the caller supplies how scripts are evaluated (in whatever isolate it has). */
168
- export async function analyzeDynamicWith(
169
- html: string,
170
- options: DynamicAnalysisOptions,
171
- evaluate: IsolatedEvaluator
172
- ): Promise<{ report: BehaviorReport; findings: Finding[] }> {
173
- const scripts = extractInlineScripts(html);
174
- const report = scripts.length ? await evaluate(scripts, options) : EMPTY_REPORT;
175
- return { report, findings: behaviorFindings(report, options.url) };
140
+ export async function analyzeDynamicWith(html, options, evaluate) {
141
+ const scripts = extractInlineScripts(html);
142
+ const report = scripts.length ? await evaluate(scripts, options) : EMPTY_REPORT;
143
+ return { report, findings: behaviorFindings(report, options.url) };
176
144
  }
177
-
178
- const EXFIL_SCORE: RuleScoreModel = { base: 72, tags: ["exfiltration", "script"] };
179
- const OFFSITE_REQUEST_SCORE: RuleScoreModel = { base: 8, tags: ["script"] };
180
- const REDIRECT_SCORE: RuleScoreModel = { base: 45, tags: ["redirect", "script"] };
145
+ const EXFIL_SCORE = { base: 72, tags: ["exfiltration", "script"] };
146
+ const OFFSITE_REQUEST_SCORE = { base: 8, tags: ["script"] };
147
+ const REDIRECT_SCORE = { base: 45, tags: ["redirect", "script"] };
181
148
  // eval/Function/string-timer use is ubiquitous in legitimate bundles — weak
182
149
  // alone. The re-scan of what they produce (below) is where real convictions come from.
183
- const EVAL_SCORE: RuleScoreModel = { base: 12, tags: ["obfuscation", "script"], maxGroup: "dynamic-code" };
184
-
150
+ const EVAL_SCORE = { base: 12, tags: ["obfuscation", "script"], maxGroup: "dynamic-code" };
185
151
  /** Map recorded behavior to scanner findings, re-scanning injected markup and decoded/eval'd code. */
186
- export function behaviorFindings(report: BehaviorReport, baseUrl?: string): Finding[] {
187
- const findings: Finding[] = [];
188
- let i = 0;
189
- const add = (severity: Severity, confidence: Confidence, model: RuleScoreModel, ruleId: string, title: string, description: string, location: string, metadata: Record<string, unknown>) => {
190
- findings.push({ id: `${ruleId}:${i++}`, ruleId, severity, confidence, score: model.base, scoreModel: model, title, description, locationType: "javascript", locationValue: location, metadata });
191
- };
192
-
193
- for (const target of report.network) {
194
- // An off-site runtime request is NOT exfiltration on its own — legit sites
195
- // constantly fetch their own subdomains, CDNs, payment processors, analytics
196
- // and APIs. Only convict (high) when the destination host ITSELF looks
197
- // suspicious (shortener, suspicious TLD, punycode, IP literal, shared/
198
- // generated host) the actual sketchy-endpoint exfil pattern. A request to
199
- // an ordinary off-site domain is recorded as a low-signal note, not a flag.
200
- if (!isThirdPartyTarget(target.url, baseUrl)) continue;
201
- const suspiciousDestination = baseUrl ? assessRedirect(baseUrl, target.url)?.destinationSuspicious ?? false : false;
202
- if (suspiciousDestination) {
203
- add("high", "high", EXFIL_SCORE, "runtime_offsite_exfil", `Runtime ${target.kind} to a suspicious off-site endpoint`, `Page JavaScript issued a ${target.kind} request to a suspicious unrelated host at runtime — a common credential/data exfiltration pattern.`, target.url, { kind: target.kind });
204
- } else {
205
- add("info", "low", OFFSITE_REQUEST_SCORE, "runtime_offsite_request", `Runtime ${target.kind} to an unrelated domain`, `Page JavaScript issued a ${target.kind} request to an unrelated (but not obviously suspicious) domain at runtime.`, target.url, { kind: target.kind });
152
+ export function behaviorFindings(report, baseUrl) {
153
+ const findings = [];
154
+ let i = 0;
155
+ const add = (severity, confidence, model, ruleId, title, description, location, metadata) => {
156
+ findings.push({ id: `${ruleId}:${i++}`, ruleId, severity, confidence, score: model.base, scoreModel: model, title, description, locationType: "javascript", locationValue: location, metadata });
157
+ };
158
+ for (const target of report.network) {
159
+ // An off-site runtime request is NOT exfiltration on its own — legit sites
160
+ // constantly fetch their own subdomains, CDNs, payment processors, analytics
161
+ // and APIs. Only convict (high) when the destination host ITSELF looks
162
+ // suspicious (shortener, suspicious TLD, punycode, IP literal, shared/
163
+ // generated host) the actual sketchy-endpoint exfil pattern. A request to
164
+ // an ordinary off-site domain is recorded as a low-signal note, not a flag.
165
+ if (!isThirdPartyTarget(target.url, baseUrl))
166
+ continue;
167
+ const suspiciousDestination = baseUrl ? assessRedirect(baseUrl, target.url)?.destinationSuspicious ?? false : false;
168
+ if (suspiciousDestination) {
169
+ add("high", "high", EXFIL_SCORE, "runtime_offsite_exfil", `Runtime ${target.kind} to a suspicious off-site endpoint`, `Page JavaScript issued a ${target.kind} request to a suspicious unrelated host at runtime — a common credential/data exfiltration pattern.`, target.url, { kind: target.kind });
170
+ }
171
+ else {
172
+ add("info", "low", OFFSITE_REQUEST_SCORE, "runtime_offsite_request", `Runtime ${target.kind} to an unrelated domain`, `Page JavaScript issued a ${target.kind} request to an unrelated (but not obviously suspicious) domain at runtime.`, target.url, { kind: target.kind });
173
+ }
206
174
  }
207
- }
208
- for (const redirect of report.redirects) {
209
- if (isThirdPartyTarget(redirect, baseUrl)) {
210
- add("medium", "high", REDIRECT_SCORE, "runtime_offsite_redirect", "JavaScript navigated to an unrelated domain at runtime", "Page JavaScript set location to an unrelated domain — used to cloak content from scanners and route victims onward.", redirect, {});
175
+ for (const redirect of report.redirects) {
176
+ if (isThirdPartyTarget(redirect, baseUrl)) {
177
+ add("medium", "high", REDIRECT_SCORE, "runtime_offsite_redirect", "JavaScript navigated to an unrelated domain at runtime", "Page JavaScript set location to an unrelated domain — used to cloak content from scanners and route victims onward.", redirect, {});
178
+ }
211
179
  }
212
- }
213
- if (report.evals.length) {
214
- add("low", "medium", EVAL_SCORE, "runtime_dynamic_code", "Runtime dynamic code execution", `Page JavaScript invoked eval/Function/string-timer ${report.evals.length} time(s) at runtime.`, "eval", { count: report.evals.length });
215
- }
216
-
217
- // Re-scan runtime-produced content (injected markup, decoded blobs, eval'd
218
- // code) through the static scanner so existing rules apply to it.
219
- const derived = [...report.writes, ...report.decoded, ...report.evals];
220
- for (const chunk of derived) {
221
- if (!chunk || chunk.length < 8) continue;
222
- const scanner = createScanner({ source: { url: baseUrl, contentType: "text/html" } });
223
- scanner.feed(new TextEncoder().encode(chunk));
224
- for (const finding of scanner.finish().findings) {
225
- findings.push({ ...finding, id: `dyn.${finding.ruleId}:${i++}`, metadata: { ...finding.metadata, via: "dynamic_analysis" } });
180
+ if (report.evals.length) {
181
+ add("low", "medium", EVAL_SCORE, "runtime_dynamic_code", "Runtime dynamic code execution", `Page JavaScript invoked eval/Function/string-timer ${report.evals.length} time(s) at runtime.`, "eval", { count: report.evals.length });
226
182
  }
227
- }
228
- return findings;
183
+ // Re-scan runtime-produced content (injected markup, decoded blobs, eval'd
184
+ // code) through the static scanner so existing rules apply to it.
185
+ const derived = [...report.writes, ...report.decoded, ...report.evals];
186
+ for (const chunk of derived) {
187
+ if (!chunk || chunk.length < 8)
188
+ continue;
189
+ const scanner = createScanner({ source: { url: baseUrl, contentType: "text/html" } });
190
+ scanner.feed(new TextEncoder().encode(chunk));
191
+ for (const finding of scanner.finish().findings) {
192
+ findings.push({ ...finding, id: `dyn.${finding.ruleId}:${i++}`, metadata: { ...finding.metadata, via: "dynamic_analysis" } });
193
+ }
194
+ }
195
+ return findings;
229
196
  }
230
-
231
197
  // URLs a page's JavaScript surfaced at runtime — redirect/navigation targets,
232
198
  // fetch/XHR/script/form endpoints, and any links embedded in markup the JS
233
199
  // injected (document.write / innerHTML). The crawler merges the same-origin
234
200
  // ones into its frontier so it CONTINUES into JS-revealed pages instead of
235
201
  // treating dynamic analysis as a dead end.
236
- export function discoveredUrlsFromBehavior(report: BehaviorReport, baseUrl?: string): string[] {
237
- const urls = new Set<string>();
238
- for (const redirect of report.redirects) if (redirect) urls.add(redirect);
239
- for (const target of report.network) if (target.url) urls.add(target.url);
240
- for (const chunk of report.writes) {
241
- if (!chunk || chunk.length < 4) continue;
242
- const scanner = createScanner({ source: { url: baseUrl, contentType: "text/html" } });
243
- scanner.feed(new TextEncoder().encode(chunk));
244
- for (const url of scanner.finish().urls) urls.add(url.normalized);
245
- }
246
- return [...urls];
202
+ export function discoveredUrlsFromBehavior(report, baseUrl) {
203
+ const urls = new Set();
204
+ for (const redirect of report.redirects)
205
+ if (redirect)
206
+ urls.add(redirect);
207
+ for (const target of report.network)
208
+ if (target.url)
209
+ urls.add(target.url);
210
+ for (const chunk of report.writes) {
211
+ if (!chunk || chunk.length < 4)
212
+ continue;
213
+ const scanner = createScanner({ source: { url: baseUrl, contentType: "text/html" } });
214
+ scanner.feed(new TextEncoder().encode(chunk));
215
+ for (const url of scanner.finish().urls)
216
+ urls.add(url.normalized);
217
+ }
218
+ return [...urls];
247
219
  }
248
-
249
220
  // A runtime target counts only if it's a different registrable domain than the
250
221
  // page AND not a known ad/analytics/CDN host — so a site's own subdomains and
251
222
  // mainstream third parties (gstatic, analytics) don't read as exfil.
252
- function isThirdPartyTarget(url: string, baseUrl?: string): boolean {
253
- let absolute: string;
254
- let targetHost: string;
255
- try {
256
- const resolved = new URL(url, baseUrl);
257
- if (resolved.protocol !== "http:" && resolved.protocol !== "https:") return false;
258
- absolute = resolved.toString();
259
- targetHost = resolved.hostname.toLowerCase();
260
- } catch {
261
- return false;
262
- }
263
- if (isAdOrAnalyticsHost(absolute)) return false;
264
- if (!baseUrl) return true;
265
- try {
266
- const baseHost = new URL(baseUrl).hostname.toLowerCase();
267
- const targetReg = registrableDomainFor(targetHost) ?? targetHost;
268
- const baseReg = registrableDomainFor(baseHost) ?? baseHost;
269
- return targetReg !== baseReg;
270
- } catch {
271
- return true;
272
- }
223
+ function isThirdPartyTarget(url, baseUrl) {
224
+ let absolute;
225
+ let targetHost;
226
+ try {
227
+ const resolved = new URL(url, baseUrl);
228
+ if (resolved.protocol !== "http:" && resolved.protocol !== "https:")
229
+ return false;
230
+ absolute = resolved.toString();
231
+ targetHost = resolved.hostname.toLowerCase();
232
+ }
233
+ catch {
234
+ return false;
235
+ }
236
+ if (isAdOrAnalyticsHost(absolute))
237
+ return false;
238
+ if (!baseUrl)
239
+ return true;
240
+ try {
241
+ const baseHost = new URL(baseUrl).hostname.toLowerCase();
242
+ const targetReg = registrableDomainFor(targetHost) ?? targetHost;
243
+ const baseReg = registrableDomainFor(baseHost) ?? baseHost;
244
+ return targetReg !== baseReg;
245
+ }
246
+ catch {
247
+ return true;
248
+ }
273
249
  }
250
+ //# sourceMappingURL=dynamic.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"dynamic.js","sourceRoot":"","sources":["../src/dynamic.ts"],"names":[],"mappings":"AAAA,wCAAwC;AACxC,EAAE;AACF,iFAAiF;AACjF,iFAAiF;AACjF,yEAAyE;AACzE,8EAA8E;AAC9E,yEAAyE;AACzE,2EAA2E;AAC3E,yEAAyE;AACzE,EAAE;AACF,0DAA0D;AAC1D,8EAA8E;AAC9E,iFAAiF;AACjF,6EAA6E;AAC7E,+EAA+E;AAC/E,8CAA8C;AAC9C,8EAA8E;AAC9E,4EAA4E;AAE5E,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,mBAAmB,EAAE,oBAAoB,EAAgD,MAAM,YAAY,CAAC;AA0BpJ,MAAM,YAAY,GAAmB,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;AAEjI,yFAAyF;AACzF,kFAAkF;AAClF,0EAA0E;AAC1E,iFAAiF;AACjF,4EAA4E;AAC5E,6CAA6C;AAC7C,MAAM,CAAC,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6DxC,CAAC;AAEF,IAAI,gBAAgB,GAAiE,IAAI,CAAC;AAC1F,SAAS,iBAAiB;IACxB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,uCAAuC;QACvC,gBAAgB,GAAG,IAAI,QAAQ,CAAC,GAAG,eAAe,0BAA0B,CAAC,EAAyD,CAAC;IACzI,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EAAE,CAAC;QAC7E,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7B,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,SAAS,CAAC,oDAAoD;QAC5F,IAAI,+EAA+E,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,SAAS;QAC1G,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QAC9B,IAAI,IAAI;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,gFAAgF;AAChF,kFAAkF;AAClF,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,oDAAoD,CAAC,EAAE,CAAC;QACxF,IAAI,KAAK,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;AAC/B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,OAAiB,EAAE,UAAkC,EAAE;IACrF,IAAI,CAAC;QACH,OAAO,iBAAiB,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IACnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,GAAG,YAAY,EAAE,MAAM,EAAE,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,EAAE,CAAC;IACnG,CAAC;AACH,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,UAAkC,EAAE;IAC/E,MAAM,MAAM,GAAG,eAAe,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;IACpE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;AACrE,CAAC;AAED,gGAAgG;AAChG,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,OAA+B,EAC/B,QAA2B;IAE3B,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;IAChF,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;AACrE,CAAC;AAED,MAAM,WAAW,GAAmB,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,CAAC;AACnF,MAAM,qBAAqB,GAAmB,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;AAC5E,MAAM,cAAc,GAAmB,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC;AAClF,4EAA4E;AAC5E,uFAAuF;AACvF,MAAM,UAAU,GAAmB,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,aAAa,EAAE,QAAQ,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC;AAE3G,sGAAsG;AACtG,MAAM,UAAU,gBAAgB,CAAC,MAAsB,EAAE,OAAgB;IACvE,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,GAAG,GAAG,CAAC,QAAkB,EAAE,UAAsB,EAAE,KAAqB,EAAE,MAAc,EAAE,KAAa,EAAE,WAAmB,EAAE,QAAgB,EAAE,QAAiC,EAAE,EAAE;QACzL,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;IACnM,CAAC,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpC,2EAA2E;QAC3E,6EAA6E;QAC7E,uEAAuE;QACvE,uEAAuE;QACvE,4EAA4E;QAC5E,4EAA4E;QAC5E,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC;YAAE,SAAS;QACvD,MAAM,qBAAqB,GAAG,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,qBAAqB,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;QACpH,IAAI,qBAAqB,EAAE,CAAC;YAC1B,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,uBAAuB,EAAE,WAAW,MAAM,CAAC,IAAI,oCAAoC,EAAE,4BAA4B,MAAM,CAAC,IAAI,qGAAqG,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACzS,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,qBAAqB,EAAE,yBAAyB,EAAE,WAAW,MAAM,CAAC,IAAI,yBAAyB,EAAE,4BAA4B,MAAM,CAAC,IAAI,4EAA4E,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAChR,CAAC;IACH,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACxC,IAAI,kBAAkB,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;YAC1C,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,0BAA0B,EAAE,wDAAwD,EAAE,qHAAqH,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;QACnQ,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,sBAAsB,EAAE,gCAAgC,EAAE,sDAAsD,MAAM,CAAC,KAAK,CAAC,MAAM,sBAAsB,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACtO,CAAC;IAED,2EAA2E;IAC3E,kEAAkE;IAClE,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QACzC,MAAM,OAAO,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;QACtF,OAAO,CAAC,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9C,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC;YAChD,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,OAAO,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC;QAChI,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,2EAA2E;AAC3E,4EAA4E;AAC5E,2EAA2E;AAC3E,2CAA2C;AAC3C,MAAM,UAAU,0BAA0B,CAAC,MAAsB,EAAE,OAAgB;IACjF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS;QAAE,IAAI,QAAQ;YAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC1E,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO;QAAE,IAAI,MAAM,CAAC,GAAG;YAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1E,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QACzC,MAAM,OAAO,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;QACtF,OAAO,CAAC,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9C,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;AACnB,CAAC;AAED,+EAA+E;AAC/E,8EAA8E;AAC9E,qEAAqE;AACrE,SAAS,kBAAkB,CAAC,GAAW,EAAE,OAAgB;IACvD,IAAI,QAAgB,CAAC;IACrB,IAAI,UAAkB,CAAC;IACvB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACvC,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAClF,QAAQ,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC/B,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,mBAAmB,CAAC,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAChD,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,SAAS,GAAG,oBAAoB,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC;QACjE,MAAM,OAAO,GAAG,oBAAoB,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC;QAC3D,OAAO,SAAS,KAAK,OAAO,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
@@ -0,0 +1,66 @@
1
+ export interface IntelStorage {
2
+ get(key: string): Promise<Uint8Array | null>;
3
+ put(key: string, value: Uint8Array): Promise<void>;
4
+ list(prefix: string): Promise<string[]>;
5
+ delete?(key: string): Promise<void>;
6
+ }
7
+ export interface FeedEntry {
8
+ host: string;
9
+ score: number;
10
+ }
11
+ export interface FeedMeta {
12
+ source?: string;
13
+ generatedAt?: string;
14
+ }
15
+ export interface FeedRecord extends FeedMeta {
16
+ version: string;
17
+ /** Distinct score bands present in this version, and the host count in each. */
18
+ bands: Record<string, number>;
19
+ }
20
+ export interface GlobalFeedManifest {
21
+ feeds: Record<string, FeedRecord>;
22
+ }
23
+ export interface CachedFeedMatch {
24
+ feedId: string;
25
+ host: string;
26
+ score: number;
27
+ source?: string;
28
+ }
29
+ /** Default score bands. Callers may use any integer band; these are conventions. */
30
+ export declare const FEED_SCORE_ACTIVE = 90;
31
+ export declare const FEED_SCORE_AGED = 55;
32
+ /** All 256 shard prefixes ("00".."ff"); a staged build finalizes one per job. */
33
+ export declare const SHARD_PREFIXES: string[];
34
+ /** Stable, synchronous host -> shard bucket. FNV-1a low byte; loader and matcher must agree. */
35
+ export declare function shardOf(host: string): string;
36
+ /** Score a feed entry from its evidence: active/recent => high, else aged/weak. */
37
+ export declare function scoreFor(input: {
38
+ active?: boolean;
39
+ addedAt?: string | null;
40
+ recencyDays?: number;
41
+ now?: number;
42
+ activeScore?: number;
43
+ agedScore?: number;
44
+ }): number;
45
+ /** Rebuild a feed's shard index from a complete entry set (feeds that fit in memory). */
46
+ export declare function rebuildFeed(storage: IntelStorage, feedId: string, version: string, entries: FeedEntry[], meta?: FeedMeta): Promise<FeedRecord>;
47
+ /** Write one download chunk's parsed entries to staging. Safe to run many in parallel. */
48
+ export declare function writeFeedChunk(storage: IntelStorage, feedId: string, version: string, chunkId: string, entries: FeedEntry[]): Promise<void>;
49
+ /** Merge every chunk's partials for one (band, prefix) into the final shard. One job per shard. */
50
+ export declare function finalizeFeedShard(storage: IntelStorage, feedId: string, version: string, band: number, prefix: string): Promise<number>;
51
+ /** Publish the feed: write its manifest, register it globally, and sweep staging + old versions. */
52
+ export declare function finalizeFeed(storage: IntelStorage, feedId: string, version: string, bands: Record<string, number>, meta?: FeedMeta): Promise<FeedRecord>;
53
+ /** Match candidate hosts against all published feeds, returning the highest score band per hit. */
54
+ export declare function matchCachedFeeds(storage: IntelStorage, hosts: string[]): Promise<CachedFeedMatch[]>;
55
+ /** Extract a lowercase host from a URL or bare host line; null for comments/blanks. */
56
+ export declare function hostFromLine(line: string): string | null;
57
+ /** OpenPhish community feed: one active phishing URL per line. */
58
+ export declare function parseOpenPhishFeed(text: string, score?: number): FeedEntry[];
59
+ /** A bare domain/host blocklist (e.g. Phishing.Database lists) at a caller-chosen score. */
60
+ export declare function parseHostList(text: string, score: number): FeedEntry[];
61
+ /** URLhaus CSV (id,dateadded,url,url_status,...): online+recent scores high, else aged. */
62
+ export declare function parseUrlhausCsv(text: string, opts?: {
63
+ recencyDays?: number;
64
+ now?: number;
65
+ }): FeedEntry[];
66
+ //# sourceMappingURL=feeds.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"feeds.d.ts","sourceRoot":"","sources":["../src/feeds.ts"],"names":[],"mappings":"AAiBA,MAAM,WAAW,YAAY;IAC3B,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC7C,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACxC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,UAAW,SAAQ,QAAQ;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,oFAAoF;AACpF,eAAO,MAAM,iBAAiB,KAAK,CAAC;AACpC,eAAO,MAAM,eAAe,KAAK,CAAC;AAQlC,iFAAiF;AACjF,eAAO,MAAM,cAAc,EAAE,MAAM,EAA2E,CAAC;AAE/G,gGAAgG;AAChG,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQ5C;AAED,mFAAmF;AACnF,wBAAgB,QAAQ,CAAC,KAAK,EAAE;IAC9B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,MAAM,CAKT;AAID,yFAAyF;AACzF,wBAAsB,WAAW,CAC/B,OAAO,EAAE,YAAY,EACrB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,SAAS,EAAE,EACpB,IAAI,GAAE,QAAa,GAClB,OAAO,CAAC,UAAU,CAAC,CAUrB;AAID,0FAA0F;AAC1F,wBAAsB,cAAc,CAClC,OAAO,EAAE,YAAY,EACrB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,SAAS,EAAE,GACnB,OAAO,CAAC,IAAI,CAAC,CAOf;AAED,mGAAmG;AACnG,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,YAAY,EACrB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,CAAC,CAUjB;AAED,oGAAoG;AACpG,wBAAsB,YAAY,CAChC,OAAO,EAAE,YAAY,EACrB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC7B,IAAI,GAAE,QAAa,GAClB,OAAO,CAAC,UAAU,CAAC,CAiBrB;AAID,mGAAmG;AACnG,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA4BzG;AAID,uFAAuF;AACvF,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAWxD;AAED,kEAAkE;AAClE,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,SAAoB,GAAG,SAAS,EAAE,CAEvF;AAED,4FAA4F;AAC5F,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,CAEtE;AAED,2FAA2F;AAC3F,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,SAAS,EAAE,CAW5G"}