@bbradar/mcp 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -2
- package/dist/bbradarClient.d.ts +7 -4
- package/dist/bbradarClient.js +41 -9
- package/dist/bbradarClient.js.map +1 -1
- package/dist/sanitize.js +14 -1
- package/dist/sanitize.js.map +1 -1
- package/dist/server.js +387 -47
- package/dist/server.js.map +1 -1
- package/package.json +1 -1
package/dist/server.js
CHANGED
|
@@ -6,7 +6,7 @@ import { NumericCursorStore } from "./cursorStore.js";
|
|
|
6
6
|
import { stableStringify, stripUndefined } from "./json.js";
|
|
7
7
|
import { FixedWindowRateLimiter } from "./rateLimit.js";
|
|
8
8
|
import { readArray, readBoolean, readNumber, readObject, readString, sanitizeJson, sanitizeChange, sanitizeProgram, sanitizeRepositoryChange, sanitizeTarget, sanitizeTargetsForExport } from "./sanitize.js";
|
|
9
|
-
const SERVER_VERSION = "0.2.
|
|
9
|
+
const SERVER_VERSION = "0.2.5";
|
|
10
10
|
const MAX_PROGRAMS_PER_SEARCH = 100;
|
|
11
11
|
const MAX_TARGET_EXPORT = 500;
|
|
12
12
|
const MAX_RECENT_CHANGES = 100;
|
|
@@ -27,6 +27,7 @@ const MAX_HUNT_TARGETS_PER_RESULT = 10;
|
|
|
27
27
|
const MAX_BULK_PROGRAM_RESOURCE_LINKS = 5;
|
|
28
28
|
const DEFAULT_RESOLVE_UPSTREAM_REQUEST_BUDGET = 12;
|
|
29
29
|
const MAX_ACCEPTED_NUMERIC_LIMIT = 1_000;
|
|
30
|
+
const MAX_UPSTREAM_PAGE = 500;
|
|
30
31
|
const PROGRAM_COLLECTION_CONCURRENCY = 4;
|
|
31
32
|
const TARGET_SAMPLE_CONCURRENCY = 3;
|
|
32
33
|
const DEFAULT_FILTER_DISCOVERY_PAGES = 2;
|
|
@@ -66,6 +67,7 @@ const CORE_TOOL_NAMES = new Set([
|
|
|
66
67
|
"export_targets"
|
|
67
68
|
]);
|
|
68
69
|
const TOOL_ENDPOINT_REQUIREMENTS = {
|
|
70
|
+
search_targets: "target_search",
|
|
69
71
|
get_repository_changes: "repository_changes",
|
|
70
72
|
find_repository_hunt_targets: "repository_changes",
|
|
71
73
|
export_targets: "target_export",
|
|
@@ -183,6 +185,8 @@ const vdpModeSchema = z.enum(["include", "exclude", "only_likely", "only_known_n
|
|
|
183
185
|
const targetMatchModeSchema = z
|
|
184
186
|
.enum(["smart", "contains", "exact", "suffix", "wildcard"])
|
|
185
187
|
.describe("How to match target_query. smart is forgiving for URLs, wildcards, target names, and partial words.");
|
|
188
|
+
const apiTargetMatchModeSchema = z.enum(["contains", "exact", "suffix", "wildcard"]);
|
|
189
|
+
const targetChangeTypeSchema = z.enum(["added", "readded", "updated", "removed"]);
|
|
186
190
|
const programNameActionSchema = z.enum(["new_targets", "targets", "scope_delta", "brief", "target_breakdown"]);
|
|
187
191
|
const guideTopicSchema = z.enum(["start", "program", "discovery", "freshness", "targets", "compare", "export", "filters"]).default("start");
|
|
188
192
|
const readOnlyAnnotations = {
|
|
@@ -241,6 +245,15 @@ const targetQuerySchema = z
|
|
|
241
245
|
})
|
|
242
246
|
.describe("Target identifier, domain, URL, wildcard, app/package/repo name, or partial target text. URLs and wildcards are normalized automatically.");
|
|
243
247
|
const isoDateTimeSchema = z.string().datetime({ offset: true });
|
|
248
|
+
const stableSnapshotSchema = z
|
|
249
|
+
.string()
|
|
250
|
+
.trim()
|
|
251
|
+
.min(1)
|
|
252
|
+
.max(1_024)
|
|
253
|
+
.refine((value) => !/[\u0000-\u001F\u007F]/.test(value), {
|
|
254
|
+
message: "Snapshot tokens cannot contain control characters."
|
|
255
|
+
})
|
|
256
|
+
.describe("Optional stable-page token. Send 'start' on page 1, then reuse meta.snapshot with identical filters. Snapshot creation is consistency-first and can be slower; omit it for the fastest live result.");
|
|
244
257
|
const pageSchema = z.number().int().min(1).default(1);
|
|
245
258
|
const programsPageSizeSchema = z.number().int().min(1).max(MAX_PROGRAMS_PER_SEARCH).default(20);
|
|
246
259
|
const recentChangesPageSizeSchema = z.number().int().min(1).max(MAX_RECENT_CHANGES).default(50);
|
|
@@ -436,6 +449,7 @@ export function createBbradarServer(client, config) {
|
|
|
436
449
|
const feedCursorStore = new NumericCursorStore(config.cursorStatePath, MAX_IN_MEMORY_FEED_CURSORS);
|
|
437
450
|
const maxProgramsPageSize = client.endpointNumber("programs", "max_page_size", MAX_PROGRAMS_PER_SEARCH);
|
|
438
451
|
const maxProgramTargetsPageSize = client.endpointNumber("program_targets", "max_page_size", MAX_TARGETS_PER_PROGRAM);
|
|
452
|
+
const maxTargetSearchPageSize = client.endpointNumber("target_search", "max_page_size", MAX_TARGETS_PER_PROGRAM);
|
|
439
453
|
const maxTargetChangesPageSize = client.endpointNumber("target_changes", "max_page_size", MAX_RECENT_CHANGES);
|
|
440
454
|
const maxRepositoryChangesPageSize = client.endpointNumber("repository_changes", "max_page_size", MAX_RECENT_CHANGES);
|
|
441
455
|
const maxTargetExport = client.endpointNumber("target_export", "max_targets", MAX_TARGET_EXPORT);
|
|
@@ -444,14 +458,16 @@ export function createBbradarServer(client, config) {
|
|
|
444
458
|
const recentChangesPageSizeSchema = z.number().int().min(1).max(maxTargetChangesPageSize).default(Math.min(50, maxTargetChangesPageSize));
|
|
445
459
|
const repositoryChangesPageSizeSchema = z.number().int().min(1).max(maxRepositoryChangesPageSize).default(Math.min(50, maxRepositoryChangesPageSize));
|
|
446
460
|
const acceptedTargetLimitSchema = z.number().int().min(1).max(maxProgramTargetsPageSize);
|
|
461
|
+
const targetSearchPageSizeSchema = z.number().int().min(1).max(maxTargetSearchPageSize).default(Math.min(50, maxTargetSearchPageSize));
|
|
447
462
|
registerTool("search_programs", {
|
|
448
463
|
title: "Search BBRadar Programs",
|
|
449
|
-
description: `Search or list active platform
|
|
464
|
+
description: `Search or list active and confirmed paused platform programs plus active Self-Hosted programs. page_size max ${maxProgramsPageSize}. Stable snapshots are opt-in because creating one can be slower than live indexed pagination.`,
|
|
450
465
|
inputSchema: {
|
|
451
466
|
search: searchTextSchema.optional().describe("Server-side program name, handle, platform, policy-host, or target search."),
|
|
452
467
|
platforms: platformListSchema(),
|
|
453
468
|
tags: tagListSchema(),
|
|
454
469
|
updated_since: isoDateTimeSchema.optional().describe("ISO datetime."),
|
|
470
|
+
snapshot: stableSnapshotSchema.optional(),
|
|
455
471
|
vdp_mode: vdpModeSchema.default("exclude").describe("VDP/no-bounty filter mode."),
|
|
456
472
|
page: pageSchema,
|
|
457
473
|
page_size: programsPageSizeSchema
|
|
@@ -460,6 +476,7 @@ export function createBbradarServer(client, config) {
|
|
|
460
476
|
annotations: readOnlyAnnotations
|
|
461
477
|
}, (args) => runTool("search_programs", config, rateLimiter, async () => {
|
|
462
478
|
const { vdp_mode, ...filters } = args;
|
|
479
|
+
assertStableSnapshotSupported(client, "programs", filters.snapshot);
|
|
463
480
|
const api = await client.listPrograms(toQuery(filters));
|
|
464
481
|
const data = readObject(api.data);
|
|
465
482
|
const programs = readArray(data?.programs)
|
|
@@ -471,6 +488,47 @@ export function createBbradarServer(client, config) {
|
|
|
471
488
|
meta: sanitizeJson(data?.meta)
|
|
472
489
|
});
|
|
473
490
|
}));
|
|
491
|
+
registerTool("search_targets", {
|
|
492
|
+
title: "Search BBRadar Targets",
|
|
493
|
+
description: `Low-level reverse target search and current target-inventory paging. q may be omitted for inventory traversal. page_size max ${maxTargetSearchPageSize}. Stable snapshots are opt-in and intentionally not used by find_hunt_targets.`,
|
|
494
|
+
inputSchema: {
|
|
495
|
+
q: targetQuerySchema.optional(),
|
|
496
|
+
match_mode: apiTargetMatchModeSchema.default("contains"),
|
|
497
|
+
snapshot: stableSnapshotSchema.optional(),
|
|
498
|
+
platforms: platformListSchema(),
|
|
499
|
+
tags: tagListSchema(),
|
|
500
|
+
include_out_of_scope: z.boolean().default(false),
|
|
501
|
+
include_ineligible: z.boolean().default(false),
|
|
502
|
+
page: pageSchema,
|
|
503
|
+
page_size: targetSearchPageSizeSchema,
|
|
504
|
+
output_mode: targetListModeSchema.default("compact")
|
|
505
|
+
},
|
|
506
|
+
outputSchema: {
|
|
507
|
+
...apiEnvelopeOutputShape(),
|
|
508
|
+
results: z.array(jsonRecordSchema()).optional(),
|
|
509
|
+
meta: jsonRecordSchema().optional(),
|
|
510
|
+
continuation: jsonRecordSchema().optional()
|
|
511
|
+
},
|
|
512
|
+
annotations: readOnlyAnnotations
|
|
513
|
+
}, (args) => runTool("search_targets", config, rateLimiter, async () => {
|
|
514
|
+
const { output_mode, ...filters } = args;
|
|
515
|
+
assertStableSnapshotSupported(client, "target_search", filters.snapshot);
|
|
516
|
+
const api = await client.searchTargets(toQuery(filters));
|
|
517
|
+
const data = readObject(api.data);
|
|
518
|
+
const results = readArray(data?.results).map((value) => {
|
|
519
|
+
const row = readObject(value) ?? {};
|
|
520
|
+
const program = sanitizeProgram(row.program, config.webBaseUrl);
|
|
521
|
+
const target = sanitizeTarget(row.target);
|
|
522
|
+
return stripUndefined({
|
|
523
|
+
program: formatProgram(program, output_mode === "full" ? "full" : "compact"),
|
|
524
|
+
target: output_mode === "full" ? target : compactTarget(target)
|
|
525
|
+
});
|
|
526
|
+
});
|
|
527
|
+
return applyCollectionContinuation(withApiMetadata(api, {
|
|
528
|
+
results,
|
|
529
|
+
meta: sanitizeJson(data?.meta)
|
|
530
|
+
}), "results", "target search results", continuationStore, config);
|
|
531
|
+
}));
|
|
474
532
|
registerTool("get_program", {
|
|
475
533
|
title: "Get BBRadar Program",
|
|
476
534
|
description: "Get one program by platform:handle program_id, handle, or human program name. Names are resolved automatically when unambiguous.",
|
|
@@ -629,7 +687,8 @@ export function createBbradarServer(client, config) {
|
|
|
629
687
|
title: "Get Recent Target Changes",
|
|
630
688
|
description: `List recent target changes. page_size max ${MAX_RECENT_CHANGES}.`,
|
|
631
689
|
inputSchema: {
|
|
632
|
-
change_type:
|
|
690
|
+
change_type: targetChangeTypeSchema.optional(),
|
|
691
|
+
snapshot: stableSnapshotSchema.optional(),
|
|
633
692
|
after_id: z.number().int().min(0).optional().describe("Incremental cursor; returns changes after this id in ascending order."),
|
|
634
693
|
resume_cursor: z.boolean().default(false).describe("Resume the in-memory cursor for this exact normalized filter set. Seed it once with after_id."),
|
|
635
694
|
max_cursor_pages: z.number().int().min(1).max(10).default(2),
|
|
@@ -658,16 +717,24 @@ export function createBbradarServer(client, config) {
|
|
|
658
717
|
annotations: readOnlyAnnotations
|
|
659
718
|
}, (args) => runTool("get_recent_changes", config, rateLimiter, async () => {
|
|
660
719
|
const { output_mode, resume_cursor, max_cursor_pages, cursor_time_budget_ms, ...filters } = args;
|
|
720
|
+
assertStableSnapshotSupported(client, "target_changes", filters.snapshot);
|
|
661
721
|
const cursor = resolveFeedCursor(feedCursorStore, "target_changes", filters, resume_cursor);
|
|
662
|
-
const { min_target_score, target_opportunity_label, competition_risk, ...upstreamFilters } = filters;
|
|
663
|
-
const collected = await collectCursorFeedPages(feedCursorStore, cursor, max_cursor_pages, cursor_time_budget_ms, (afterId) => client.getRecentChanges(toQuery({
|
|
722
|
+
const { min_target_score, target_opportunity_label, competition_risk, snapshot, ...upstreamFilters } = filters;
|
|
723
|
+
const collected = await collectCursorFeedPages(feedCursorStore, cursor, max_cursor_pages, cursor_time_budget_ms, snapshot, (afterId, recoveryProbe, activeSnapshot) => client.getRecentChanges(toQuery({
|
|
664
724
|
...upstreamFilters,
|
|
665
725
|
minTargetScore: min_target_score,
|
|
666
726
|
targetOpportunityLabel: target_opportunity_label,
|
|
667
727
|
competitionRisk: competition_risk,
|
|
728
|
+
snapshot: activeSnapshot,
|
|
668
729
|
after_id: afterId,
|
|
669
730
|
page: afterId === undefined ? filters.page : 1
|
|
670
|
-
})))
|
|
731
|
+
}), recoveryProbe ? { cache: false, coalesce: false } : {}), (startPage, recoveryProbe) => refreshInventoryBaseline((page, pageProbe) => client.searchTargets(toQuery({
|
|
732
|
+
q: upstreamFilters.search,
|
|
733
|
+
platforms: upstreamFilters.platforms,
|
|
734
|
+
tags: upstreamFilters.tags,
|
|
735
|
+
page,
|
|
736
|
+
page_size: client.endpointNumber("target_search", "max_page_size", 100)
|
|
737
|
+
}), pageProbe ? { cache: false, coalesce: false } : {}), "results", max_cursor_pages, cursor_time_budget_ms, startPage, recoveryProbe));
|
|
671
738
|
const api = collected.lastApi;
|
|
672
739
|
const data = collected.lastData;
|
|
673
740
|
const sanitizedMeta = readObject(sanitizeJson(data?.meta));
|
|
@@ -682,7 +749,9 @@ export function createBbradarServer(client, config) {
|
|
|
682
749
|
pages_fetched: collected.pagesFetched,
|
|
683
750
|
returned_across_pages: changes.length,
|
|
684
751
|
cursor_page_budget_exhausted: collected.pageBudgetExhausted,
|
|
685
|
-
cursor_time_budget_exhausted: collected.timeBudgetExhausted
|
|
752
|
+
cursor_time_budget_exhausted: collected.timeBudgetExhausted,
|
|
753
|
+
history_gap: collected.gapRecovery !== undefined,
|
|
754
|
+
inventory_baseline: collected.gapRecovery
|
|
686
755
|
}),
|
|
687
756
|
facets: sanitizeJson(data?.facets),
|
|
688
757
|
cursor: collected.cursor,
|
|
@@ -694,6 +763,7 @@ export function createBbradarServer(client, config) {
|
|
|
694
763
|
description: "Read-only repository push/change feed for direct inspection or polling. Defaults to lightweight program summaries; use after_id for efficient incremental polling. Use find_repository_hunt_targets instead when reward or target-quality ranking is also required.",
|
|
695
764
|
inputSchema: {
|
|
696
765
|
after_id: z.number().int().min(0).optional().describe("Incremental cursor; returns events after this id in ascending order."),
|
|
766
|
+
snapshot: stableSnapshotSchema.optional(),
|
|
697
767
|
resume_cursor: z.boolean().default(false).describe("Resume the in-memory cursor for this exact normalized filter set. Seed it once with after_id."),
|
|
698
768
|
max_cursor_pages: z.number().int().min(1).max(10).default(2),
|
|
699
769
|
cursor_time_budget_ms: z.number().int().min(100).max(10_000).default(3_000),
|
|
@@ -724,12 +794,20 @@ export function createBbradarServer(client, config) {
|
|
|
724
794
|
throw new Error("The negotiated BBRadar API contract does not expose repository changes.");
|
|
725
795
|
}
|
|
726
796
|
const { output_mode, resume_cursor, max_cursor_pages, cursor_time_budget_ms, ...filters } = args;
|
|
797
|
+
assertStableSnapshotSupported(client, "repository_changes", filters.snapshot);
|
|
727
798
|
const cursor = resolveFeedCursor(feedCursorStore, "repository_changes", filters, resume_cursor);
|
|
728
|
-
const
|
|
729
|
-
|
|
799
|
+
const { snapshot, ...upstreamFilters } = filters;
|
|
800
|
+
const collected = await collectCursorFeedPages(feedCursorStore, cursor, max_cursor_pages, cursor_time_budget_ms, snapshot, (afterId, recoveryProbe, activeSnapshot) => client.getRepositoryChanges(toQuery({
|
|
801
|
+
...upstreamFilters,
|
|
802
|
+
snapshot: activeSnapshot,
|
|
730
803
|
after_id: afterId,
|
|
731
804
|
page: afterId === undefined ? filters.page : 1
|
|
732
|
-
})))
|
|
805
|
+
}), recoveryProbe ? { cache: false, coalesce: false } : {}), (startPage, recoveryProbe) => refreshInventoryBaseline((page, pageProbe) => client.getRepositoryChanges(toQuery({
|
|
806
|
+
...upstreamFilters,
|
|
807
|
+
after_id: undefined,
|
|
808
|
+
page,
|
|
809
|
+
page_size: client.endpointNumber("repository_changes", "max_page_size", 100)
|
|
810
|
+
}), pageProbe ? { cache: false, coalesce: false } : {}), "results", max_cursor_pages, cursor_time_budget_ms, startPage, recoveryProbe));
|
|
733
811
|
const api = collected.lastApi;
|
|
734
812
|
const data = collected.lastData;
|
|
735
813
|
return applyCollectionContinuation(withApiMetadata(api, {
|
|
@@ -740,7 +818,9 @@ export function createBbradarServer(client, config) {
|
|
|
740
818
|
pages_fetched: collected.pagesFetched,
|
|
741
819
|
returned_across_pages: collected.results.length,
|
|
742
820
|
cursor_page_budget_exhausted: collected.pageBudgetExhausted,
|
|
743
|
-
cursor_time_budget_exhausted: collected.timeBudgetExhausted
|
|
821
|
+
cursor_time_budget_exhausted: collected.timeBudgetExhausted,
|
|
822
|
+
history_gap: collected.gapRecovery !== undefined,
|
|
823
|
+
inventory_baseline: collected.gapRecovery
|
|
744
824
|
}),
|
|
745
825
|
facets: sanitizeJson(data?.facets),
|
|
746
826
|
available_facets: sanitizeJson(data?.available_facets),
|
|
@@ -750,7 +830,7 @@ export function createBbradarServer(client, config) {
|
|
|
750
830
|
}));
|
|
751
831
|
registerTool("find_hunt_targets", {
|
|
752
832
|
title: "Find Best Hunt Targets",
|
|
753
|
-
description: "Preferred one-call path for requests combining good rewards, broad Web2/Web3 scope, target quality, competition, and recently added targets. Returns a final ranked shortlist with exact bounded target samples and recent additions; do not fan out into repeated find_programs/get_program_targets calls. Use find_repository_hunt_targets instead only when Git or repository activity is explicitly requested.",
|
|
833
|
+
description: "Preferred one-call path for requests combining good rewards, broad Web2/Web3 scope, target quality, competition, and recently added targets. Returns a final ranked shortlist with exact bounded target samples and recent additions; do not fan out into repeated find_programs/get_program_targets calls. Self-Hosted is supported through current inventory and automatically skips its unavailable change feed. Use find_repository_hunt_targets instead only when Git or repository activity is explicitly requested.",
|
|
754
834
|
inputSchema: {
|
|
755
835
|
platforms: platformListSchema(),
|
|
756
836
|
exclude_platforms: stringListSchema("Platforms to exclude from the final shortlist."),
|
|
@@ -795,7 +875,7 @@ export function createBbradarServer(client, config) {
|
|
|
795
875
|
}, (args) => runTool("find_hunt_targets", config, rateLimiter, async () => findHuntTargets(client, config, args)));
|
|
796
876
|
registerTool("find_repository_hunt_targets", {
|
|
797
877
|
title: "Find Repository-Aware Hunt Targets",
|
|
798
|
-
description: "Fast one-call path for requests that combine recent repository activity with program rewards and target quality. Fetches program rankings and repository changes in parallel, joins them locally, and works with arbitrary
|
|
878
|
+
description: "Fast one-call path for requests that combine recent repository activity with program rewards and target quality. Fetches program rankings and repository changes in parallel, joins them locally, and works with arbitrary supported platform catalogs, Web2/Web3 modes, reward thresholds, target types, languages, and freshness criteria. Self-Hosted has no repository stream and is rejected without upstream work.",
|
|
799
879
|
inputSchema: {
|
|
800
880
|
platforms: platformListSchema(),
|
|
801
881
|
exclude_platforms: stringListSchema("Platforms to exclude from final matches."),
|
|
@@ -916,7 +996,7 @@ export function createBbradarServer(client, config) {
|
|
|
916
996
|
inputSchema: {
|
|
917
997
|
program_id: programIdentifierSchema,
|
|
918
998
|
since: isoDateTimeSchema.optional(),
|
|
919
|
-
change_type:
|
|
999
|
+
change_type: targetChangeTypeSchema.optional(),
|
|
920
1000
|
target_type: filterValueSchema().optional(),
|
|
921
1001
|
language_tags: languageTagListSchema(),
|
|
922
1002
|
include_removed: z.boolean().default(true),
|
|
@@ -940,7 +1020,7 @@ export function createBbradarServer(client, config) {
|
|
|
940
1020
|
title: "Get Recent Target Activity",
|
|
941
1021
|
description: "Recent target changes grouped by program.",
|
|
942
1022
|
inputSchema: {
|
|
943
|
-
change_type:
|
|
1023
|
+
change_type: targetChangeTypeSchema.optional(),
|
|
944
1024
|
target_type: filterValueSchema().optional(),
|
|
945
1025
|
include_removed: z.boolean().default(true),
|
|
946
1026
|
include_out_of_scope: z.boolean().default(false),
|
|
@@ -1426,7 +1506,7 @@ export function createBbradarServer(client, config) {
|
|
|
1426
1506
|
description: "Recent changes by target type grouped by program.",
|
|
1427
1507
|
inputSchema: {
|
|
1428
1508
|
target_type: filterValueSchema(),
|
|
1429
|
-
change_type:
|
|
1509
|
+
change_type: targetChangeTypeSchema.default("added"),
|
|
1430
1510
|
include_removed: z.boolean().default(false),
|
|
1431
1511
|
include_out_of_scope: z.boolean().default(false),
|
|
1432
1512
|
include_ineligible: z.boolean().default(false),
|
|
@@ -4407,21 +4487,50 @@ function latestAddedProgramMatches(change, input, platformFilters) {
|
|
|
4407
4487
|
return true;
|
|
4408
4488
|
}
|
|
4409
4489
|
async function getProgramTargetsPayload(client, input, programId) {
|
|
4410
|
-
const
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
|
|
4416
|
-
|
|
4417
|
-
|
|
4418
|
-
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4490
|
+
const pageSize = client.endpointNumber("program_targets", "max_page_size", MAX_TARGETS_PER_PROGRAM);
|
|
4491
|
+
// `eligibility=eligible` is an API-level in-scope bounty-eligibility filter,
|
|
4492
|
+
// so the common eligible-only path needs no second local scope pass.
|
|
4493
|
+
const directOffsetPaging = !input.include_ineligible || (!input.strict_scope_filter && input.include_out_of_scope);
|
|
4494
|
+
const collected = [];
|
|
4495
|
+
let matchedSeen = directOffsetPaging ? Math.floor(input.offset / pageSize) * pageSize : 0;
|
|
4496
|
+
let page = directOffsetPaging ? Math.floor(input.offset / pageSize) + 1 : 1;
|
|
4497
|
+
const firstPage = page;
|
|
4498
|
+
let api;
|
|
4499
|
+
let upstreamMeta;
|
|
4500
|
+
let upstreamHasMore = false;
|
|
4501
|
+
do {
|
|
4502
|
+
api = await client.getProgramTargets(programId, {
|
|
4503
|
+
search: input.search,
|
|
4504
|
+
target_type: input.target_type,
|
|
4505
|
+
eligibility: input.include_ineligible ? "all" : "eligible",
|
|
4506
|
+
minTargetScore: input.min_target_score,
|
|
4507
|
+
targetOpportunityLabel: input.target_opportunity_label,
|
|
4508
|
+
competitionRisk: input.competition_risk,
|
|
4509
|
+
page,
|
|
4510
|
+
page_size: pageSize
|
|
4511
|
+
});
|
|
4512
|
+
const data = readObject(api.data);
|
|
4513
|
+
upstreamMeta = readObject(data?.meta);
|
|
4514
|
+
for (const target of readArray(data?.targets).map(sanitizeTarget)) {
|
|
4515
|
+
if (!targetHasAllowedScope(target, input))
|
|
4516
|
+
continue;
|
|
4517
|
+
if (matchedSeen >= input.offset && collected.length < input.limit)
|
|
4518
|
+
collected.push(target);
|
|
4519
|
+
matchedSeen += 1;
|
|
4520
|
+
}
|
|
4521
|
+
upstreamHasMore = readBoolean(upstreamMeta?.has_more) === true;
|
|
4522
|
+
const totalPages = readNumber(upstreamMeta?.total_pages);
|
|
4523
|
+
page += 1;
|
|
4524
|
+
if (totalPages !== undefined && page > totalPages) {
|
|
4525
|
+
upstreamHasMore = false;
|
|
4526
|
+
}
|
|
4527
|
+
} while (upstreamHasMore &&
|
|
4528
|
+
page <= MAX_UPSTREAM_PAGE &&
|
|
4529
|
+
collected.length < input.limit);
|
|
4530
|
+
if (!api) {
|
|
4531
|
+
throw new Error("No program target response was collected.");
|
|
4532
|
+
}
|
|
4533
|
+
const targets = collected;
|
|
4425
4534
|
return withApiMetadata(api, {
|
|
4426
4535
|
program_id: programId,
|
|
4427
4536
|
targets: formatTargetList(targets, input.output_mode),
|
|
@@ -4429,12 +4538,16 @@ async function getProgramTargetsPayload(client, input, programId) {
|
|
|
4429
4538
|
offset: input.offset,
|
|
4430
4539
|
limit: input.limit,
|
|
4431
4540
|
output_mode: input.output_mode,
|
|
4432
|
-
total_active_targets: readNumber(upstreamMeta?.total) ??
|
|
4433
|
-
total_after_filters:
|
|
4541
|
+
total_active_targets: readNumber(upstreamMeta?.total) ?? matchedSeen,
|
|
4542
|
+
total_after_filters: upstreamHasMore ? undefined : matchedSeen,
|
|
4434
4543
|
returned: targets.length,
|
|
4435
|
-
has_more: input.offset +
|
|
4436
|
-
upstream_has_more:
|
|
4437
|
-
possibly_incomplete:
|
|
4544
|
+
has_more: upstreamHasMore || matchedSeen > input.offset + targets.length,
|
|
4545
|
+
upstream_has_more: upstreamHasMore,
|
|
4546
|
+
possibly_incomplete: upstreamHasMore,
|
|
4547
|
+
upstream_pages_fetched: page - firstPage,
|
|
4548
|
+
upstream_start_page: firstPage,
|
|
4549
|
+
direct_offset_paging: directOffsetPaging,
|
|
4550
|
+
upstream_page_limit_reached: upstreamHasMore && page > MAX_UPSTREAM_PAGE,
|
|
4438
4551
|
data_capabilities: readObject(upstreamMeta?.data_capabilities),
|
|
4439
4552
|
target_source: "program_targets"
|
|
4440
4553
|
}
|
|
@@ -5789,6 +5902,8 @@ function compactProgram(program) {
|
|
|
5789
5902
|
name: stringField(program, "name"),
|
|
5790
5903
|
platform: stringField(program, "platform"),
|
|
5791
5904
|
catalog: stringField(program, "catalog"),
|
|
5905
|
+
program_status: stringField(program, "program_status"),
|
|
5906
|
+
is_active: readBoolean(program.is_active),
|
|
5792
5907
|
program_type: stringField(program, "program_type"),
|
|
5793
5908
|
platform_url: stringField(program, "platform_url"),
|
|
5794
5909
|
bbradar_url: stringField(program, "bbradar_url"),
|
|
@@ -5807,6 +5922,9 @@ function compactProgram(program) {
|
|
|
5807
5922
|
last_updated: stringField(program, "last_updated"),
|
|
5808
5923
|
targets_updated_at: stringField(program, "targets_updated_at"),
|
|
5809
5924
|
scope_summary: readObject(program.scope_summary),
|
|
5925
|
+
match_qualities: Array.isArray(program.match_qualities)
|
|
5926
|
+
? readStringArrayField(program, "match_qualities")
|
|
5927
|
+
: undefined,
|
|
5810
5928
|
resource_links: readObject(program.resource_links)
|
|
5811
5929
|
});
|
|
5812
5930
|
}
|
|
@@ -5867,6 +5985,7 @@ function formatRepositoryChange(change, mode) {
|
|
|
5867
5985
|
})
|
|
5868
5986
|
: undefined,
|
|
5869
5987
|
changes_url: stringField(change, "changes_url"),
|
|
5988
|
+
comparison: sanitizeJson(change.comparison),
|
|
5870
5989
|
programs: readArray(change.programs).map((program) => formatProgram(readObject(program) ?? {}, "compact")),
|
|
5871
5990
|
targets: Array.isArray(change.targets)
|
|
5872
5991
|
? change.targets.map((target) => compactTarget(readObject(target) ?? {}))
|
|
@@ -6351,7 +6470,11 @@ function clampLimit(name, value, max, warnings) {
|
|
|
6351
6470
|
async function findHuntTargets(client, config, input) {
|
|
6352
6471
|
const warnings = [];
|
|
6353
6472
|
const errors = [];
|
|
6354
|
-
const
|
|
6473
|
+
const selfHostedOnly = isSelfHostedOnlyPlatformSelection(input.platforms);
|
|
6474
|
+
const targetChangesEndpointSupported = client.supportsEndpoint("target_changes");
|
|
6475
|
+
const recentChangesEnabled = input.include_recent_changes &&
|
|
6476
|
+
!selfHostedOnly &&
|
|
6477
|
+
targetChangesEndpointSupported;
|
|
6355
6478
|
const desiredRecentChangeQueries = recentChangesEnabled ? huntRecentChangeQueries(input) : [];
|
|
6356
6479
|
const recentChangeQueries = desiredRecentChangeQueries.slice(0, Math.max(0, input.upstream_request_budget - 2));
|
|
6357
6480
|
const recentRequestReserve = recentChangeQueries.length;
|
|
@@ -6399,7 +6522,10 @@ async function findHuntTargets(client, config, input) {
|
|
|
6399
6522
|
output_mode: "compact",
|
|
6400
6523
|
upstream_request_budget: programRequestBudget
|
|
6401
6524
|
};
|
|
6402
|
-
if (input.include_recent_changes &&
|
|
6525
|
+
if (input.include_recent_changes && selfHostedOnly) {
|
|
6526
|
+
warnings.push("Self-Hosted does not expose target-change history; recent target evidence was derived only from current target-inventory timestamps.");
|
|
6527
|
+
}
|
|
6528
|
+
else if (input.include_recent_changes && !recentChangesEnabled) {
|
|
6403
6529
|
warnings.push("The negotiated API contract does not expose target changes; ranked current programs without recent-change enrichment.");
|
|
6404
6530
|
}
|
|
6405
6531
|
else if (recentChangeQueries.length < desiredRecentChangeQueries.length) {
|
|
@@ -6502,6 +6628,7 @@ async function findHuntTargets(client, config, input) {
|
|
|
6502
6628
|
exact_target_programs_sampled: targetResult.targetsByProgram.size,
|
|
6503
6629
|
recent_changes_scanned: recentResult.changes.length,
|
|
6504
6630
|
recent_programs_matched: recentByProgram.size,
|
|
6631
|
+
recent_change_catalog_supported: !selfHostedOnly && targetChangesEndpointSupported,
|
|
6505
6632
|
model_round_trips_expected: 1,
|
|
6506
6633
|
parallel_recent_change_fetch: recentChangeQueries.length > 0,
|
|
6507
6634
|
bulk_resource_links_suppressed: true,
|
|
@@ -6747,7 +6874,9 @@ function buildHuntTargetMatch(entry, targetsByProgram, recentByProgram, input) {
|
|
|
6747
6874
|
(readNumber(rankFactors.target_surface_score) ?? 0) * 0.25);
|
|
6748
6875
|
const freshnessScore = huntFreshnessScore(latestRecentAt, recentTargets.length, readNumber(rankFactors.added_7d));
|
|
6749
6876
|
const publicReports = readNumber(rankFactors.public_report_count);
|
|
6750
|
-
|
|
6877
|
+
// Missing report evidence is unknown, not low competition. Keep the
|
|
6878
|
+
// internal ranking conservative and do not expose a made-up numeric signal.
|
|
6879
|
+
const competitionScore = publicReports === undefined ? 20 : Math.max(0, 100 - Math.min(100, publicReports / 10));
|
|
6751
6880
|
const bestTargetScore = Math.max(0, ...targetSamples.map((target) => readNumber(target.target_opportunity_score) ?? 0));
|
|
6752
6881
|
const baseHuntScore = readNumber(rankFactors.hunt_value_score) ?? 0;
|
|
6753
6882
|
const compositeScore = huntCompositeScore(input.ranking_mode, baseHuntScore, rewardScore, scopeScore, freshnessScore, competitionScore, bestTargetScore);
|
|
@@ -6775,7 +6904,8 @@ function buildHuntTargetMatch(entry, targetsByProgram, recentByProgram, input) {
|
|
|
6775
6904
|
reward_score: roundScore(rewardScore),
|
|
6776
6905
|
scope_score: roundScore(scopeScore),
|
|
6777
6906
|
freshness_score: roundScore(freshnessScore),
|
|
6778
|
-
competition_score: roundScore(competitionScore),
|
|
6907
|
+
competition_score: publicReports === undefined ? undefined : roundScore(competitionScore),
|
|
6908
|
+
competition_signal: publicReports === undefined ? "unknown" : "public_report_count",
|
|
6779
6909
|
best_sample_target_score: bestTargetScore > 0 ? bestTargetScore : undefined,
|
|
6780
6910
|
recent_added_targets_in_window: recentTargets.length,
|
|
6781
6911
|
latest_added_target_at: latestRecentAt,
|
|
@@ -6880,11 +7010,52 @@ function compareHuntTargetMatches(left, right, mode) {
|
|
|
6880
7010
|
}
|
|
6881
7011
|
async function findRepositoryHuntTargets(client, config, input) {
|
|
6882
7012
|
const warnings = [];
|
|
7013
|
+
const repositoryPlatforms = input.platforms.filter((platform) => !isSelfHostedPlatform(platform));
|
|
7014
|
+
const selfHostedRequested = repositoryPlatforms.length !== input.platforms.length;
|
|
7015
|
+
if (input.platforms.length > 0 && repositoryPlatforms.length === 0) {
|
|
7016
|
+
return {
|
|
7017
|
+
request_id: randomUUID(),
|
|
7018
|
+
fetched_at: new Date().toISOString(),
|
|
7019
|
+
source_requests: [],
|
|
7020
|
+
warnings: [
|
|
7021
|
+
"Self-Hosted does not expose repository-change events or target intelligence, so repository-aware hunting is unavailable for this catalog. Use find_hunt_targets for current Self-Hosted program and target inventory."
|
|
7022
|
+
],
|
|
7023
|
+
filters: {
|
|
7024
|
+
platforms: input.platforms,
|
|
7025
|
+
surface_mode: input.surface_mode,
|
|
7026
|
+
ranking_mode: input.ranking_mode,
|
|
7027
|
+
output_mode: input.output_mode
|
|
7028
|
+
},
|
|
7029
|
+
upstream_budget: {
|
|
7030
|
+
initial: input.upstream_request_budget,
|
|
7031
|
+
used: 0,
|
|
7032
|
+
remaining: input.upstream_request_budget
|
|
7033
|
+
},
|
|
7034
|
+
ranking_scope: {
|
|
7035
|
+
mode: "unsupported_catalog",
|
|
7036
|
+
possibly_incomplete: false
|
|
7037
|
+
},
|
|
7038
|
+
programs: [],
|
|
7039
|
+
matches: [],
|
|
7040
|
+
meta: {
|
|
7041
|
+
returned: 0,
|
|
7042
|
+
matched_programs: 0,
|
|
7043
|
+
program_candidates: 0,
|
|
7044
|
+
repository_events_scanned: 0,
|
|
7045
|
+
qualifying_targets_before_result_limit: 0,
|
|
7046
|
+
repository_request_attempts: 0,
|
|
7047
|
+
parallel_source_fetch: false
|
|
7048
|
+
}
|
|
7049
|
+
};
|
|
7050
|
+
}
|
|
7051
|
+
if (selfHostedRequested) {
|
|
7052
|
+
warnings.push("Self-Hosted was excluded from repository-aware matching because that catalog has no repository-change stream; other requested platforms were processed normally.");
|
|
7053
|
+
}
|
|
6883
7054
|
const repositoryRequestReserve = input.upstream_request_budget >= 3 ? 2 : 1;
|
|
6884
7055
|
const programRequestBudget = Math.max(1, input.upstream_request_budget - repositoryRequestReserve);
|
|
6885
7056
|
const programInput = {
|
|
6886
|
-
platforms:
|
|
6887
|
-
exclude_platforms: input.exclude_platforms,
|
|
7057
|
+
platforms: repositoryPlatforms,
|
|
7058
|
+
exclude_platforms: uniqueStrings([...input.exclude_platforms, "Self-Hosted"]),
|
|
6888
7059
|
tags: input.tags,
|
|
6889
7060
|
scope_tags: input.scope_tags,
|
|
6890
7061
|
target_types: input.target_types,
|
|
@@ -6926,7 +7097,7 @@ async function findRepositoryHuntTargets(client, config, input) {
|
|
|
6926
7097
|
};
|
|
6927
7098
|
const [programPayload, repository] = await Promise.all([
|
|
6928
7099
|
findPrograms(client, config, programInput),
|
|
6929
|
-
fetchRepositoryHuntPage(client, input, warnings, repositoryRequestReserve)
|
|
7100
|
+
fetchRepositoryHuntPage(client, { ...input, platforms: repositoryPlatforms }, warnings, repositoryRequestReserve)
|
|
6930
7101
|
]);
|
|
6931
7102
|
const candidatePrograms = readArray(programPayload.programs)
|
|
6932
7103
|
.map(readObject)
|
|
@@ -7737,6 +7908,10 @@ function toProgramCandidate(rawProgram, webBaseUrl) {
|
|
|
7737
7908
|
return candidate;
|
|
7738
7909
|
}
|
|
7739
7910
|
function programCandidateMatches(candidate, filters) {
|
|
7911
|
+
const programStatus = normalizeTag(stringField(candidate.program, "program_status") ?? "");
|
|
7912
|
+
if (readBoolean(candidate.program.is_active) === false || programStatus === "paused") {
|
|
7913
|
+
return false;
|
|
7914
|
+
}
|
|
7740
7915
|
if (filters.platforms.length > 0 && !filters.platforms.map(normalizeTag).includes(normalizeTag(stringField(candidate.program, "platform") ?? ""))) {
|
|
7741
7916
|
return false;
|
|
7742
7917
|
}
|
|
@@ -8062,6 +8237,12 @@ function uniqueStrings(values) {
|
|
|
8062
8237
|
function normalizeTag(value) {
|
|
8063
8238
|
return value.trim().toLowerCase();
|
|
8064
8239
|
}
|
|
8240
|
+
function isSelfHostedPlatform(value) {
|
|
8241
|
+
return normalizeTag(value).replace(/[\s_]+/g, "-") === "self-hosted";
|
|
8242
|
+
}
|
|
8243
|
+
function isSelfHostedOnlyPlatformSelection(platforms) {
|
|
8244
|
+
return platforms.length > 0 && platforms.every(isSelfHostedPlatform);
|
|
8245
|
+
}
|
|
8065
8246
|
function tokenSet(value) {
|
|
8066
8247
|
return new Set(value.split(/[^a-z0-9]+/).filter((token) => token.length >= 2));
|
|
8067
8248
|
}
|
|
@@ -8172,7 +8353,7 @@ function advanceFeedCursor(store, cursor, rawMeta) {
|
|
|
8172
8353
|
stored_in_memory: true
|
|
8173
8354
|
};
|
|
8174
8355
|
}
|
|
8175
|
-
async function collectCursorFeedPages(store, initialCursor, maxPages, timeBudgetMs, fetchPage) {
|
|
8356
|
+
async function collectCursorFeedPages(store, initialCursor, maxPages, timeBudgetMs, initialSnapshot, fetchPage, refreshInventory) {
|
|
8176
8357
|
const startedAt = Date.now();
|
|
8177
8358
|
const results = [];
|
|
8178
8359
|
const sourceRequests = [];
|
|
@@ -8184,9 +8365,110 @@ async function collectCursorFeedPages(store, initialCursor, maxPages, timeBudget
|
|
|
8184
8365
|
let hasMore = false;
|
|
8185
8366
|
let pageBudgetExhausted = false;
|
|
8186
8367
|
let timeBudgetExhausted = false;
|
|
8368
|
+
let gapRecovery;
|
|
8369
|
+
let activeSnapshot = initialSnapshot;
|
|
8187
8370
|
do {
|
|
8188
|
-
|
|
8371
|
+
let api;
|
|
8372
|
+
try {
|
|
8373
|
+
api = await fetchPage(currentCursor.afterId, undefined, activeSnapshot);
|
|
8374
|
+
feedRecoveryProgress.get(store)?.delete(currentCursor.key);
|
|
8375
|
+
}
|
|
8376
|
+
catch (error) {
|
|
8377
|
+
if (!(error instanceof BBRadarApiError) ||
|
|
8378
|
+
error.status !== 410 ||
|
|
8379
|
+
currentCursor.afterId === undefined ||
|
|
8380
|
+
activeSnapshot !== undefined ||
|
|
8381
|
+
!refreshInventory) {
|
|
8382
|
+
throw error;
|
|
8383
|
+
}
|
|
8384
|
+
const recoveryNonce = `${Date.now()}-${currentCursor.afterId}`;
|
|
8385
|
+
const markerApi = await fetchPage(undefined, `${recoveryNonce}-marker`);
|
|
8386
|
+
const markerData = readObject(markerApi.data);
|
|
8387
|
+
const markerAfterId = feedBaselineAfterId(markerData);
|
|
8388
|
+
let progress = feedRecoveryProgress.get(store)?.get(currentCursor.key);
|
|
8389
|
+
let restartedForDrift = false;
|
|
8390
|
+
if (progress && (progress.previousAfterId !== currentCursor.afterId ||
|
|
8391
|
+
progress.baselineAfterId !== markerAfterId)) {
|
|
8392
|
+
feedRecoveryProgress.get(store)?.delete(currentCursor.key);
|
|
8393
|
+
progress = undefined;
|
|
8394
|
+
restartedForDrift = true;
|
|
8395
|
+
}
|
|
8396
|
+
const beforeAfterId = progress?.baselineAfterId ?? markerAfterId;
|
|
8397
|
+
const startPage = progress?.nextPage ?? 1;
|
|
8398
|
+
const inventoryPart = await refreshInventory(startPage, `${recoveryNonce}-inventory`);
|
|
8399
|
+
const inventory = {
|
|
8400
|
+
...inventoryPart,
|
|
8401
|
+
rows_observed: (progress?.rowsObserved ?? 0) + (readNumber(inventoryPart.rows_observed) ?? 0),
|
|
8402
|
+
pages_fetched: (progress?.pagesFetched ?? 0) + (readNumber(inventoryPart.pages_fetched) ?? 0),
|
|
8403
|
+
resumed_from_page: startPage,
|
|
8404
|
+
restarted_for_feed_drift: restartedForDrift
|
|
8405
|
+
};
|
|
8406
|
+
const inventoryComplete = readBoolean(inventory.complete) === true;
|
|
8407
|
+
const endpointPageLimitReached = readBoolean(inventory.endpoint_page_limit_reached) === true;
|
|
8408
|
+
const afterApi = inventoryComplete ? await fetchPage(undefined, `${recoveryNonce}-after`) : markerApi;
|
|
8409
|
+
const afterData = readObject(afterApi.data);
|
|
8410
|
+
const afterAfterId = feedBaselineAfterId(afterData);
|
|
8411
|
+
const baselineStable = inventoryComplete && beforeAfterId !== undefined && beforeAfterId === afterAfterId;
|
|
8412
|
+
const nextAfterId = baselineStable ? beforeAfterId : currentCursor.afterId;
|
|
8413
|
+
if (baselineStable) {
|
|
8414
|
+
store.set(currentCursor.key, nextAfterId);
|
|
8415
|
+
feedRecoveryProgress.get(store)?.delete(currentCursor.key);
|
|
8416
|
+
}
|
|
8417
|
+
else if (!inventoryComplete && !endpointPageLimitReached && beforeAfterId !== undefined) {
|
|
8418
|
+
// Persist only the unchanged expired bookmark so resume_cursor can
|
|
8419
|
+
// continue recovery without treating partial inventory as a baseline.
|
|
8420
|
+
store.set(currentCursor.key, currentCursor.afterId);
|
|
8421
|
+
setRecoveryCheckpoint(store, currentCursor.key, {
|
|
8422
|
+
previousAfterId: currentCursor.afterId,
|
|
8423
|
+
baselineAfterId: beforeAfterId,
|
|
8424
|
+
nextPage: readNumber(inventoryPart.next_page) ?? startPage,
|
|
8425
|
+
rowsObserved: readNumber(inventory.rows_observed) ?? 0,
|
|
8426
|
+
pagesFetched: readNumber(inventory.pages_fetched) ?? 0
|
|
8427
|
+
});
|
|
8428
|
+
}
|
|
8429
|
+
else {
|
|
8430
|
+
// A failed first explicit recovery must remain resumable even when no
|
|
8431
|
+
// safe inventory checkpoint can be retained.
|
|
8432
|
+
store.set(currentCursor.key, currentCursor.afterId);
|
|
8433
|
+
feedRecoveryProgress.get(store)?.delete(currentCursor.key);
|
|
8434
|
+
}
|
|
8435
|
+
gapRecovery = {
|
|
8436
|
+
historical_gap: true,
|
|
8437
|
+
current_inventory_refreshed: inventoryComplete,
|
|
8438
|
+
recovery_complete: baselineStable,
|
|
8439
|
+
retained_previous_cursor: !baselineStable,
|
|
8440
|
+
incomplete_reason: !inventoryComplete
|
|
8441
|
+
? endpointPageLimitReached ? "inventory_endpoint_page_limit_reached" : "inventory_page_or_time_budget_exhausted"
|
|
8442
|
+
: beforeAfterId === undefined
|
|
8443
|
+
? "feed_baseline_unavailable"
|
|
8444
|
+
: beforeAfterId !== afterAfterId
|
|
8445
|
+
? "feed_changed_during_inventory_refresh"
|
|
8446
|
+
: undefined,
|
|
8447
|
+
inventory,
|
|
8448
|
+
next_inventory_page: baselineStable ? undefined : readNumber(inventoryPart.next_page),
|
|
8449
|
+
previous_after_id: currentCursor.afterId,
|
|
8450
|
+
baseline_after_id: baselineStable ? nextAfterId : undefined,
|
|
8451
|
+
baseline_events_emitted: 0,
|
|
8452
|
+
limitation: "Changes retired before recovery cannot be reconstructed."
|
|
8453
|
+
};
|
|
8454
|
+
lastApi = afterApi;
|
|
8455
|
+
lastData = afterData;
|
|
8456
|
+
cursorPayload = {
|
|
8457
|
+
endpoint: currentCursor.endpoint,
|
|
8458
|
+
mode: baselineStable ? "recovered_baseline" : "recovery_incomplete",
|
|
8459
|
+
used_after_id: currentCursor.afterId,
|
|
8460
|
+
next_after_id: nextAfterId,
|
|
8461
|
+
has_more: false,
|
|
8462
|
+
stored_in_memory: true
|
|
8463
|
+
};
|
|
8464
|
+
pagesFetched += 1;
|
|
8465
|
+
break;
|
|
8466
|
+
}
|
|
8189
8467
|
const data = readObject(api.data);
|
|
8468
|
+
const returnedSnapshot = readString(readObject(data?.meta)?.snapshot);
|
|
8469
|
+
if (activeSnapshot === "start" && returnedSnapshot) {
|
|
8470
|
+
activeSnapshot = returnedSnapshot;
|
|
8471
|
+
}
|
|
8190
8472
|
const pageResults = readArray(data?.results);
|
|
8191
8473
|
lastApi = api;
|
|
8192
8474
|
lastData = data;
|
|
@@ -8229,12 +8511,61 @@ async function collectCursorFeedPages(store, initialCursor, maxPages, timeBudget
|
|
|
8229
8511
|
pagesFetched,
|
|
8230
8512
|
pageBudgetExhausted,
|
|
8231
8513
|
timeBudgetExhausted,
|
|
8232
|
-
sourceRequests
|
|
8514
|
+
sourceRequests,
|
|
8515
|
+
gapRecovery
|
|
8516
|
+
};
|
|
8517
|
+
}
|
|
8518
|
+
const feedRecoveryProgress = new WeakMap();
|
|
8519
|
+
function recoveryProgressFor(store) {
|
|
8520
|
+
let progress = feedRecoveryProgress.get(store);
|
|
8521
|
+
if (!progress) {
|
|
8522
|
+
progress = new Map();
|
|
8523
|
+
feedRecoveryProgress.set(store, progress);
|
|
8524
|
+
}
|
|
8525
|
+
return progress;
|
|
8526
|
+
}
|
|
8527
|
+
function setRecoveryCheckpoint(store, key, checkpoint) {
|
|
8528
|
+
const progress = recoveryProgressFor(store);
|
|
8529
|
+
if (!progress.has(key) && progress.size >= 500) {
|
|
8530
|
+
const oldest = progress.keys().next().value;
|
|
8531
|
+
if (oldest !== undefined)
|
|
8532
|
+
progress.delete(oldest);
|
|
8533
|
+
}
|
|
8534
|
+
progress.delete(key);
|
|
8535
|
+
progress.set(key, checkpoint);
|
|
8536
|
+
}
|
|
8537
|
+
function feedBaselineAfterId(data) {
|
|
8538
|
+
const metaAfterId = readNumber(readObject(data?.meta)?.next_after_id);
|
|
8539
|
+
const ids = readArray(data?.results)
|
|
8540
|
+
.map((entry) => readNumber(readObject(entry)?.id))
|
|
8541
|
+
.filter((value) => value !== undefined);
|
|
8542
|
+
return ids.length > 0 ? Math.max(metaAfterId ?? 0, ...ids) : metaAfterId;
|
|
8543
|
+
}
|
|
8544
|
+
async function refreshInventoryBaseline(fetchPage, collection, maxPages, timeBudgetMs, startPage = 1, recoveryProbe = "inventory") {
|
|
8545
|
+
const startedAt = Date.now();
|
|
8546
|
+
let page = startPage;
|
|
8547
|
+
let rows = 0;
|
|
8548
|
+
let hasMore = false;
|
|
8549
|
+
do {
|
|
8550
|
+
const api = await fetchPage(page, `${recoveryProbe}-${page}`);
|
|
8551
|
+
const data = readObject(api.data);
|
|
8552
|
+
rows += readArray(data?.[collection]).length;
|
|
8553
|
+
hasMore = readBoolean(readObject(data?.meta)?.has_more) === true;
|
|
8554
|
+
page += 1;
|
|
8555
|
+
} while (hasMore && page < startPage + maxPages && Date.now() - startedAt < timeBudgetMs);
|
|
8556
|
+
return {
|
|
8557
|
+
collection,
|
|
8558
|
+
rows_observed: rows,
|
|
8559
|
+
pages_fetched: page - startPage,
|
|
8560
|
+
next_page: hasMore && page <= MAX_UPSTREAM_PAGE ? page : undefined,
|
|
8561
|
+
endpoint_page_limit_reached: hasMore && page > MAX_UPSTREAM_PAGE,
|
|
8562
|
+
complete: !hasMore,
|
|
8563
|
+
bounded: true
|
|
8233
8564
|
};
|
|
8234
8565
|
}
|
|
8235
8566
|
function normalizeFeedCursorFilters(filters) {
|
|
8236
8567
|
return Object.fromEntries(Object.entries(filters)
|
|
8237
|
-
.filter(([key, value]) => key !== "after_id" && key !== "page" && value !== undefined)
|
|
8568
|
+
.filter(([key, value]) => key !== "after_id" && key !== "page" && key !== "snapshot" && value !== undefined)
|
|
8238
8569
|
.map(([key, value]) => [key, normalizeFeedCursorFilterValue(value)]));
|
|
8239
8570
|
}
|
|
8240
8571
|
function normalizeFeedCursorFilterValue(value) {
|
|
@@ -8441,6 +8772,11 @@ function splitFilterList(value) {
|
|
|
8441
8772
|
function toQuery(values) {
|
|
8442
8773
|
return compactRecord(values);
|
|
8443
8774
|
}
|
|
8775
|
+
function assertStableSnapshotSupported(client, endpoint, snapshot) {
|
|
8776
|
+
if (snapshot !== undefined && !client.supportsStableSnapshot(endpoint)) {
|
|
8777
|
+
throw new Error(`The negotiated BBRadar API contract does not advertise stable snapshots for ${endpoint}. Omit snapshot to use live pagination.`);
|
|
8778
|
+
}
|
|
8779
|
+
}
|
|
8444
8780
|
function compactRecord(values) {
|
|
8445
8781
|
return Object.fromEntries(Object.entries(values).filter(([, value]) => {
|
|
8446
8782
|
if (value === undefined || value === null) {
|
|
@@ -8489,6 +8825,10 @@ function compactTarget(target) {
|
|
|
8489
8825
|
in_scope: readBoolean(target.in_scope),
|
|
8490
8826
|
eligible_for_bounty: readBoolean(target.eligible_for_bounty),
|
|
8491
8827
|
wildcard: readBoolean(target.wildcard),
|
|
8828
|
+
path_kind: stringField(target, "path_kind"),
|
|
8829
|
+
path_prefix: stringField(target, "path_prefix"),
|
|
8830
|
+
match_quality: stringField(target, "match_quality"),
|
|
8831
|
+
match_reason: stringField(target, "match_reason"),
|
|
8492
8832
|
target_opportunity_score: readNumber(target.target_opportunity_score),
|
|
8493
8833
|
target_opportunity_label: stringField(target, "target_opportunity_label"),
|
|
8494
8834
|
competition_risk: stringField(target, "competition_risk"),
|