fluncle 0.264.0 → 0.265.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/fluncle.mjs +1347 -13
- package/package.json +1 -1
package/bin/fluncle.mjs
CHANGED
|
@@ -380,6 +380,12 @@ var require_main = __commonJS(function(exports, module) {
|
|
|
380
380
|
});
|
|
381
381
|
|
|
382
382
|
// src/env.ts
|
|
383
|
+
var exports_env = {};
|
|
384
|
+
__export(exports_env, {
|
|
385
|
+
getApiBaseUrl: () => getApiBaseUrl,
|
|
386
|
+
loadEnv: () => loadEnv,
|
|
387
|
+
readOptionalEnv: () => readOptionalEnv
|
|
388
|
+
});
|
|
383
389
|
import { homedir } from "node:os";
|
|
384
390
|
import { join } from "node:path";
|
|
385
391
|
function loadEnv(keys) {
|
|
@@ -399,6 +405,10 @@ function loadEnv(keys) {
|
|
|
399
405
|
}
|
|
400
406
|
return result;
|
|
401
407
|
}
|
|
408
|
+
function readOptionalEnv(key) {
|
|
409
|
+
loadConfig();
|
|
410
|
+
return process.env[key] || undefined;
|
|
411
|
+
}
|
|
402
412
|
function getApiBaseUrl() {
|
|
403
413
|
loadConfig();
|
|
404
414
|
return (process.env.FLUNCLE_API_BASE_URL ?? "https://www.fluncle.com").replace(/\/+$/, "");
|
|
@@ -622,7 +632,7 @@ function parseVersion(version) {
|
|
|
622
632
|
var currentVersion;
|
|
623
633
|
var init_version = __esm(() => {
|
|
624
634
|
init_output();
|
|
625
|
-
currentVersion = "0.
|
|
635
|
+
currentVersion = "0.265.0".trim() ? "0.265.0".trim() : "0.1.0";
|
|
626
636
|
});
|
|
627
637
|
|
|
628
638
|
// src/update-notifier.ts
|
|
@@ -2335,12 +2345,12 @@ function parseVersion2(version) {
|
|
|
2335
2345
|
var currentVersion2, latestReleaseUrl = "https://api.github.com/repos/mauricekleine/fluncle/releases/latest";
|
|
2336
2346
|
var init_version2 = __esm(() => {
|
|
2337
2347
|
init_output();
|
|
2338
|
-
currentVersion2 = "0.
|
|
2348
|
+
currentVersion2 = "0.265.0".trim() ? "0.265.0".trim() : "0.1.0";
|
|
2339
2349
|
});
|
|
2340
2350
|
|
|
2341
2351
|
// ../../packages/registry/src/index.ts
|
|
2342
2352
|
function liveSurfaces() {
|
|
2343
|
-
return SURFACES.filter((surface) => surface.pending !== true);
|
|
2353
|
+
return SURFACES.filter((surface) => surface.pending !== true && surface.dormant === undefined);
|
|
2344
2354
|
}
|
|
2345
2355
|
var SITE = "https://www.fluncle.com", PROBE_CADENCE_MS, PROBE_TIMEOUT_MS, MINUTE_MS, SURFACES;
|
|
2346
2356
|
var init_src = __esm(() => {
|
|
@@ -3524,12 +3534,13 @@ var init_src = __esm(() => {
|
|
|
3524
3534
|
weights: { status: "hidden" }
|
|
3525
3535
|
},
|
|
3526
3536
|
{
|
|
3537
|
+
dormant: "the iOS app has no users, and the mirror's embedded-replica sync of the main database halves the primary's write capacity while it runs",
|
|
3527
3538
|
exposedContent: [
|
|
3528
3539
|
"mirror the anchored, explicitly allowlisted public catalogue into the shared read-only device replica"
|
|
3529
3540
|
],
|
|
3530
3541
|
kind: "cron",
|
|
3531
3542
|
name: "cron.device-mirror",
|
|
3532
|
-
operatorNotes: "hourly, run by a host systemd timer (docs/agents/hermes/device-mirror-timer/). Explicitly synchronizes one restart-safe local embedded replica, materializes the anchored track IDs once, derives and validates one allowlisted generation locally, then stages bounded changes and atomically cuts the complete verified generation into the shared device database. Sync, derivation, upload, validation, or cutover failure leaves the previous generation visible; rebuilding the target database remains forbidden because it would reset the libSQL replication log and force every device to bootstrap again. A schema-version mismatch stops for an operator migration. Zero LLM tokens. Source: docs/agents/hermes/scripts/device-mirror.*.",
|
|
3543
|
+
operatorNotes: "DORMANT: parked by docs/agents/hermes/device-mirror-timer/DORMANT, which keeps the installer from enabling its timer and keeps the healthcheck, /status, and the run-ledger roster from expecting it; revival steps are in that directory's README. When live: hourly, run by a host systemd timer (docs/agents/hermes/device-mirror-timer/). Explicitly synchronizes one restart-safe local embedded replica, materializes the anchored track IDs once, derives and validates one allowlisted generation locally, then stages bounded changes and atomically cuts the complete verified generation into the shared device database. Sync, derivation, upload, validation, or cutover failure leaves the previous generation visible; rebuilding the target database remains forbidden because it would reset the libSQL replication log and force every device to bootstrap again. A schema-version mismatch stops for an operator migration. Zero LLM tokens. Source: docs/agents/hermes/scripts/device-mirror.*.",
|
|
3533
3544
|
probeConfig: {
|
|
3534
3545
|
cadenceMs: 60 * MINUTE_MS,
|
|
3535
3546
|
cronName: "fluncle-device-mirror",
|
|
@@ -3663,11 +3674,11 @@ var init_src = __esm(() => {
|
|
|
3663
3674
|
{
|
|
3664
3675
|
command: "fluncle admin tracks context --queue",
|
|
3665
3676
|
exposedContent: [
|
|
3666
|
-
"Firecrawl facts → distilled context_note + a Texture: line (Worker-side
|
|
3677
|
+
"Firecrawl facts → distilled context_note + a Texture: line (Worker-side OpenRouter distill)"
|
|
3667
3678
|
],
|
|
3668
3679
|
kind: "cron",
|
|
3669
3680
|
name: "cron.context-note",
|
|
3670
|
-
operatorNotes: "every 5m. --no-agent trigger; the Worker does the Firecrawl +
|
|
3681
|
+
operatorNotes: "every 5m. --no-agent trigger; the Worker does the Firecrawl + OpenRouter distill. Zero on-box tokens.",
|
|
3671
3682
|
probeConfig: { cadenceMs: 5 * MINUTE_MS, cronName: "fluncle-context-note", kind: "cron" },
|
|
3672
3683
|
statusDescription: "distills the facts behind each finding",
|
|
3673
3684
|
title: "Context notes",
|
|
@@ -3723,6 +3734,23 @@ var init_src = __esm(() => {
|
|
|
3723
3734
|
title: "Label triage gate",
|
|
3724
3735
|
weights: { status: "hidden" }
|
|
3725
3736
|
},
|
|
3737
|
+
{
|
|
3738
|
+
exposedContent: [
|
|
3739
|
+
"nightly label-outlier review list — every album and label-less single of embedded catalogue tracks scored against the rest of its own label, the far ones flagged onto /admin/label-outliers (--no-agent)"
|
|
3740
|
+
],
|
|
3741
|
+
kind: "cron",
|
|
3742
|
+
name: "cron.label-outliers",
|
|
3743
|
+
operatorNotes: "05:30 Amsterdam daily, retry slot 06:30. Reads its scoring inputs (embedded catalogue tracks with their raw vector bytes, artist credits, album Discogs styles) from the AGENT-tier list_label_outlier_inputs op in keyset pages of 1,000, each page its own admitted heavy-read phase, into a local file it scores and deletes (about 51 pages and 280 MB a night at 50k tracks; an admission yield mid-walk leaves the payload unstarted for the retry slot). It scores each album's mean cosine to its label's leave-it-out centroid as a robust z against that label's own spread (labels under 8 albums use the whole catalogue), and keeps the ones at z <= -4 whose artists have fewer than 3 typical tracks elsewhere and whose album Discogs does not file under Drum n Bass or Jungle. A corpus under 10,000 usable vectors or under 95% usable, or more than 2,000 flags, fails loudly and writes nothing. It then fires the AGENT-tier record_label_outliers op in its own admitted phase; the Worker refuses a duplicated, partial, or collapsed run (fewer than 80% of the live embedded catalogue), replaces the stored list, and returns every visible unit not yet announced. The sweep posts one Discord summary and only then acknowledges them by unit and fingerprint (acknowledge_label_outlier_alerts), so a lost response re-alerts next run. Dismissing is operator tier (set_label_outliers_dismissed) and holds until the album's tracks change. Zero LLM tokens. Source: docs/agents/hermes/scripts/label-outliers*.ts. See docs/catalogue-crawler.md.",
|
|
3744
|
+
probeConfig: {
|
|
3745
|
+
cadenceMs: 24 * 60 * MINUTE_MS,
|
|
3746
|
+
cronName: "fluncle-label-outliers",
|
|
3747
|
+
kind: "cron",
|
|
3748
|
+
schedule: { time: "05:30", tz: "Europe/Amsterdam" }
|
|
3749
|
+
},
|
|
3750
|
+
statusDescription: "flags albums that sound unlike their label",
|
|
3751
|
+
title: "Label outliers",
|
|
3752
|
+
weights: { status: "hidden" }
|
|
3753
|
+
},
|
|
3726
3754
|
{
|
|
3727
3755
|
command: "fluncle admin albums describe --queue",
|
|
3728
3756
|
exposedContent: [
|
|
@@ -3874,7 +3902,7 @@ var init_src = __esm(() => {
|
|
|
3874
3902
|
],
|
|
3875
3903
|
kind: "cron",
|
|
3876
3904
|
name: "cron.healthcheck",
|
|
3877
|
-
operatorNotes: "every 10m, run by a rave-02 host systemd timer (docs/agents/hermes/healthcheck-timer/) — decoupled from the Hermes cron gateway so the prober isn't starved by the scheduler it monitors. Pure probing, zero LLM tokens. POSTs to the agent-tier record_health op that /status reads.",
|
|
3905
|
+
operatorNotes: "every 10m, run by a rave-02 host systemd timer (docs/agents/hermes/healthcheck-timer/) — decoupled from the Hermes cron gateway so the prober isn't starved by the scheduler it monitors. Pure probing, zero LLM tokens. POSTs to the agent-tier record_health op that /status reads, and makes one record_database_write_probe call per tick that feeds database admission's write-latency guardrail.",
|
|
3878
3906
|
probeConfig: { cadenceMs: 10 * MINUTE_MS, cronName: "fluncle-healthcheck", kind: "cron" },
|
|
3879
3907
|
statusDescription: "the prober behind this very page",
|
|
3880
3908
|
title: "Healthcheck prober",
|
|
@@ -4020,6 +4048,21 @@ var init_src = __esm(() => {
|
|
|
4020
4048
|
title: "Funnel snapshot",
|
|
4021
4049
|
weights: { status: "hidden" }
|
|
4022
4050
|
},
|
|
4051
|
+
{
|
|
4052
|
+
exposedContent: [
|
|
4053
|
+
"Turso usage and overage reading — the organisation's current-cycle usage priced against the plan and projected to the reset, stored in the telemetry database for the /admin/costs Turso panel (--no-agent)"
|
|
4054
|
+
],
|
|
4055
|
+
kind: "cron",
|
|
4056
|
+
name: "cron.turso-usage",
|
|
4057
|
+
operatorNotes: "Every 6 hours (OnUnitActiveSec), no retry slot: the counters are cumulative, so the next tick loses nothing. Reads the Turso Platform API (organization usage, database list, subscription, upcoming invoice) with TURSO_PLATFORM_API_TOKEN + TURSO_PLATFORM_ORG from the sweep secrets file, then fires the AGENT-tier record_turso_usage op. The Worker prices the reading with the versioned table in apps/web/src/lib/turso-pricing.ts (list rates, so an upper bound), projects the UTC-month cycle from the recent daily run-rate, stores it in the telemetry database, and claims a (cycle, level) alert row when the projected overage reaches the turso_usage_alert_threshold_usd setting (default $50) or twice it. The sweep posts one Discord message and only then acknowledges the levels (acknowledge_turso_usage_alerts), so a lost post re-sends next run. Missing credentials fail the run by name. Zero LLM tokens. Source: docs/agents/hermes/scripts/turso-usage-sweep.*. See docs/agents/hermes/turso-usage-timer/README.md.",
|
|
4058
|
+
operatorOnly: "the database bill is an operator concern; the job still reports to the prober, the run ledger, and the /admin/costs Turso panel, never to the public status board",
|
|
4059
|
+
probeConfig: {
|
|
4060
|
+
cadenceMs: 6 * 60 * MINUTE_MS,
|
|
4061
|
+
cronName: "fluncle-turso-usage",
|
|
4062
|
+
kind: "cron"
|
|
4063
|
+
},
|
|
4064
|
+
weights: {}
|
|
4065
|
+
},
|
|
4023
4066
|
{
|
|
4024
4067
|
exposedContent: [
|
|
4025
4068
|
"daily per-post social-metrics snapshot — appends each published post's Postiz reach (views/likes/comments/…) into an append-only ledger, one row per post per day (--no-agent)"
|
|
@@ -4039,7 +4082,7 @@ var init_src = __esm(() => {
|
|
|
4039
4082
|
},
|
|
4040
4083
|
{
|
|
4041
4084
|
exposedContent: [
|
|
4042
|
-
"nightly codebase audit — one domain/night on
|
|
4085
|
+
"nightly codebase audit — one domain/night on an 8-domain rotation; opens a PR the reviewer merges (claude -p, subscription auth)"
|
|
4043
4086
|
],
|
|
4044
4087
|
kind: "cron",
|
|
4045
4088
|
name: "cron.audit",
|
|
@@ -10497,7 +10540,8 @@ var init_admin_catalogue = __esm(() => {
|
|
|
10497
10540
|
limit: number2().int().min(1).max(MAX_CRAWL_PREPARE_LIMIT).default(2),
|
|
10498
10541
|
maxHop: number2().int().min(0).max(3).default(2),
|
|
10499
10542
|
phase: literal("prepare"),
|
|
10500
|
-
sampleStorableRepair: boolean2().optional()
|
|
10543
|
+
sampleStorableRepair: boolean2().optional(),
|
|
10544
|
+
skipFrontierPendingCount: boolean2().optional()
|
|
10501
10545
|
}),
|
|
10502
10546
|
object({
|
|
10503
10547
|
phase: literal("fetch"),
|
|
@@ -10522,7 +10566,7 @@ var init_admin_catalogue = __esm(() => {
|
|
|
10522
10566
|
object({
|
|
10523
10567
|
boxFetch: boolean2(),
|
|
10524
10568
|
capabilities: CrawlPhaseCapabilitiesSchema.optional(),
|
|
10525
|
-
frontierPending: number2(),
|
|
10569
|
+
frontierPending: number2().optional(),
|
|
10526
10570
|
initialization: CrawlPhaseInitializationSchema,
|
|
10527
10571
|
items: array(object({
|
|
10528
10572
|
fetchPlan: CrawlFetchPlanSchema,
|
|
@@ -12459,7 +12503,7 @@ var init_admin_costs = __esm(() => {
|
|
|
12459
12503
|
});
|
|
12460
12504
|
|
|
12461
12505
|
// ../../packages/contracts/src/orpc/admin-database-admission.ts
|
|
12462
|
-
var DatabaseAdmissionActionSchema, DatabaseAdmissionLaneSchema, DatabaseAdmissionOutcomeSchema, DatabaseAdmissionYieldReasonSchema, DatabaseAdmissionInputSchema, DatabaseAdmissionResponseSchema, coordinateDatabaseAdmission, adminDatabaseAdmissionContract;
|
|
12506
|
+
var DatabaseAdmissionActionSchema, DatabaseAdmissionLaneSchema, DatabaseAdmissionOutcomeSchema, DatabaseAdmissionYieldReasonSchema, DatabaseAdmissionInputSchema, DatabaseAdmissionResponseSchema, coordinateDatabaseAdmission, DatabaseWriteProbeOutcomeSchema, DatabaseWriteProbeResponseSchema, recordDatabaseWriteProbe, adminDatabaseAdmissionContract;
|
|
12463
12507
|
var init_admin_database_admission = __esm(() => {
|
|
12464
12508
|
init_dist3();
|
|
12465
12509
|
init_zod();
|
|
@@ -12478,12 +12522,15 @@ var init_admin_database_admission = __esm(() => {
|
|
|
12478
12522
|
"database-health",
|
|
12479
12523
|
"direct-read-latency",
|
|
12480
12524
|
"public-latency",
|
|
12481
|
-
"queue"
|
|
12525
|
+
"queue",
|
|
12526
|
+
"write-latency"
|
|
12482
12527
|
]);
|
|
12483
12528
|
DatabaseAdmissionInputSchema = object({
|
|
12484
12529
|
action: DatabaseAdmissionActionSchema,
|
|
12485
12530
|
fencingToken: number2().int().positive().optional(),
|
|
12531
|
+
notAfterMs: number2().int().nonnegative().optional(),
|
|
12486
12532
|
owner: string2().min(1).max(64).regex(/^[a-z0-9][a-z0-9.-]*$/),
|
|
12533
|
+
protocolVersion: literal(2).optional(),
|
|
12487
12534
|
runId: string2().min(1).max(96).regex(/^[A-Za-z0-9][A-Za-z0-9._:-]*$/)
|
|
12488
12535
|
}).refine((input) => input.action === "acquire" || input.action === "cancel" || input.fencingToken !== undefined, "heartbeat and release require a fencing token");
|
|
12489
12536
|
DatabaseAdmissionResponseSchema = object({
|
|
@@ -12495,10 +12542,12 @@ var init_admin_database_admission = __esm(() => {
|
|
|
12495
12542
|
holdMs: number2().int().nonnegative(),
|
|
12496
12543
|
lane: DatabaseAdmissionLaneSchema,
|
|
12497
12544
|
leaseExpiresAtMs: number2().int().nonnegative().nullable(),
|
|
12545
|
+
leaseRemainingMs: number2().int().nonnegative().nullable(),
|
|
12498
12546
|
operationId: string2().min(1).max(64),
|
|
12499
12547
|
outcome: DatabaseAdmissionOutcomeSchema,
|
|
12500
12548
|
queueAgeMs: number2().int().nonnegative(),
|
|
12501
12549
|
recovered: boolean2(),
|
|
12550
|
+
retryAfterMs: number2().int().nonnegative().nullable(),
|
|
12502
12551
|
waitMs: number2().int().nonnegative(),
|
|
12503
12552
|
yieldReason: DatabaseAdmissionYieldReasonSchema.nullable()
|
|
12504
12553
|
});
|
|
@@ -12509,8 +12558,23 @@ var init_admin_database_admission = __esm(() => {
|
|
|
12509
12558
|
summary: "Coordinate one registry-classified recurring database operation",
|
|
12510
12559
|
tags: ["Admin"]
|
|
12511
12560
|
}).input(DatabaseAdmissionInputSchema).output(DatabaseAdmissionResponseSchema);
|
|
12561
|
+
DatabaseWriteProbeOutcomeSchema = _enum(["failed", "ok", "slow", "stalled"]);
|
|
12562
|
+
DatabaseWriteProbeResponseSchema = object({
|
|
12563
|
+
latencyMs: number2().int().nonnegative().nullable(),
|
|
12564
|
+
ok: literal(true),
|
|
12565
|
+
outcome: DatabaseWriteProbeOutcomeSchema,
|
|
12566
|
+
recorded: boolean2()
|
|
12567
|
+
});
|
|
12568
|
+
recordDatabaseWriteProbe = oc.route({
|
|
12569
|
+
method: "POST",
|
|
12570
|
+
operationId: "recordDatabaseWriteProbe",
|
|
12571
|
+
path: "/admin/database-admission/write-probe",
|
|
12572
|
+
summary: "Time one bounded write to the primary database and record it for admission",
|
|
12573
|
+
tags: ["Admin"]
|
|
12574
|
+
}).input(object({})).output(DatabaseWriteProbeResponseSchema);
|
|
12512
12575
|
adminDatabaseAdmissionContract = {
|
|
12513
|
-
coordinate_database_admission: coordinateDatabaseAdmission
|
|
12576
|
+
coordinate_database_admission: coordinateDatabaseAdmission,
|
|
12577
|
+
record_database_write_probe: recordDatabaseWriteProbe
|
|
12514
12578
|
};
|
|
12515
12579
|
});
|
|
12516
12580
|
|
|
@@ -13824,6 +13888,142 @@ var init_admin_hub_counts = __esm(() => {
|
|
|
13824
13888
|
};
|
|
13825
13889
|
});
|
|
13826
13890
|
|
|
13891
|
+
// ../../packages/contracts/src/orpc/admin-label-outliers.ts
|
|
13892
|
+
var MAX_RECORDED_LABEL_OUTLIERS = 2000, MAX_LABEL_OUTLIER_DISMISSALS = 500, LabelOutlierReferenceSchema, RecordedLabelOutlierSchema, LabelOutlierRunSchema, AlertedLabelOutlierSchema, PendingLabelOutlierAlertSchema, recordLabelOutliers, acknowledgeLabelOutlierAlerts, LabelOutlierTrackSchema, LabelOutlierItemSchema, listLabelOutliers, setLabelOutliersDismissed, LabelOutlierInputTrackSchema, LabelOutlierInputAlbumSchema, listLabelOutlierInputs, adminLabelOutliersContract;
|
|
13893
|
+
var init_admin_label_outliers = __esm(() => {
|
|
13894
|
+
init_dist3();
|
|
13895
|
+
init_zod();
|
|
13896
|
+
LabelOutlierReferenceSchema = _enum(["catalogue", "label"]);
|
|
13897
|
+
RecordedLabelOutlierSchema = object({
|
|
13898
|
+
albumId: string2().min(1).max(256).nullable(),
|
|
13899
|
+
artistSupport: number2().int().min(0),
|
|
13900
|
+
fingerprint: string2().min(1).max(128),
|
|
13901
|
+
labelId: string2().min(1).max(256).nullable(),
|
|
13902
|
+
reference: LabelOutlierReferenceSchema,
|
|
13903
|
+
referenceMedian: number2(),
|
|
13904
|
+
score: number2(),
|
|
13905
|
+
singleTrackId: string2().min(1).max(256).nullable(),
|
|
13906
|
+
trackCount: number2().int().min(1),
|
|
13907
|
+
unitId: string2().min(1).max(256),
|
|
13908
|
+
z: number2()
|
|
13909
|
+
}).meta({ id: "RecordedLabelOutlier" });
|
|
13910
|
+
LabelOutlierRunSchema = object({
|
|
13911
|
+
flagged: number2().int().min(0),
|
|
13912
|
+
labelsScored: number2().int().min(0),
|
|
13913
|
+
ranAt: string2(),
|
|
13914
|
+
replicaSyncedAt: string2().nullable(),
|
|
13915
|
+
tracksScored: number2().int().min(0),
|
|
13916
|
+
unitsScored: number2().int().min(0)
|
|
13917
|
+
}).meta({ id: "LabelOutlierRun" });
|
|
13918
|
+
AlertedLabelOutlierSchema = object({
|
|
13919
|
+
fingerprint: string2().min(1).max(128),
|
|
13920
|
+
unitId: string2().min(1).max(256)
|
|
13921
|
+
}).meta({ id: "AlertedLabelOutlier" });
|
|
13922
|
+
PendingLabelOutlierAlertSchema = object({
|
|
13923
|
+
albumName: string2().nullable(),
|
|
13924
|
+
labelName: string2().nullable(),
|
|
13925
|
+
title: string2(),
|
|
13926
|
+
unitId: string2()
|
|
13927
|
+
}).meta({ id: "PendingLabelOutlierAlert" });
|
|
13928
|
+
recordLabelOutliers = oc.route({
|
|
13929
|
+
method: "PUT",
|
|
13930
|
+
operationId: "recordLabelOutliers",
|
|
13931
|
+
path: "/admin/label-outliers",
|
|
13932
|
+
summary: "Replace the nightly label-outlier review list with the box's scored set",
|
|
13933
|
+
tags: ["Admin"]
|
|
13934
|
+
}).input(object({
|
|
13935
|
+
labelsScored: number2().int().min(0),
|
|
13936
|
+
outliers: array(RecordedLabelOutlierSchema).max(MAX_RECORDED_LABEL_OUTLIERS),
|
|
13937
|
+
replicaSyncedAt: string2().nullable(),
|
|
13938
|
+
totalFlagged: number2().int().min(0),
|
|
13939
|
+
tracksScored: number2().int().min(0),
|
|
13940
|
+
unitsScored: number2().int().min(0)
|
|
13941
|
+
})).output(object({
|
|
13942
|
+
flagged: number2().int(),
|
|
13943
|
+
ok: literal(true),
|
|
13944
|
+
pendingAlertUnits: array(AlertedLabelOutlierSchema),
|
|
13945
|
+
pendingAlerts: array(PendingLabelOutlierAlertSchema),
|
|
13946
|
+
removed: number2().int()
|
|
13947
|
+
}));
|
|
13948
|
+
acknowledgeLabelOutlierAlerts = oc.route({
|
|
13949
|
+
method: "PUT",
|
|
13950
|
+
operationId: "acknowledgeLabelOutlierAlerts",
|
|
13951
|
+
path: "/admin/label-outliers/alerts",
|
|
13952
|
+
summary: "Mark flagged label outliers as announced, once their Discord summary has landed",
|
|
13953
|
+
tags: ["Admin"]
|
|
13954
|
+
}).input(object({
|
|
13955
|
+
units: array(AlertedLabelOutlierSchema).min(1).max(MAX_RECORDED_LABEL_OUTLIERS)
|
|
13956
|
+
})).output(object({ acknowledged: number2().int(), ok: literal(true) }));
|
|
13957
|
+
LabelOutlierTrackSchema = object({
|
|
13958
|
+
artists: array(object({ name: string2(), slug: string2() })),
|
|
13959
|
+
title: string2(),
|
|
13960
|
+
trackId: string2()
|
|
13961
|
+
}).meta({ id: "LabelOutlierTrack" });
|
|
13962
|
+
LabelOutlierItemSchema = object({
|
|
13963
|
+
album: object({ id: string2(), name: string2(), slug: string2() }).nullable(),
|
|
13964
|
+
artistSupport: number2(),
|
|
13965
|
+
discogsStyles: array(string2()),
|
|
13966
|
+
dismissedAt: string2().nullable(),
|
|
13967
|
+
firstFlaggedAt: string2(),
|
|
13968
|
+
label: object({ id: string2(), name: string2(), slug: string2() }).nullable(),
|
|
13969
|
+
reference: LabelOutlierReferenceSchema,
|
|
13970
|
+
referenceMedian: number2(),
|
|
13971
|
+
score: number2(),
|
|
13972
|
+
trackCount: number2(),
|
|
13973
|
+
tracks: array(LabelOutlierTrackSchema),
|
|
13974
|
+
unitId: string2(),
|
|
13975
|
+
z: number2()
|
|
13976
|
+
}).meta({ id: "LabelOutlierItem" });
|
|
13977
|
+
listLabelOutliers = oc.route({
|
|
13978
|
+
method: "GET",
|
|
13979
|
+
operationId: "listLabelOutliers",
|
|
13980
|
+
path: "/admin/label-outliers",
|
|
13981
|
+
summary: "The label-outlier review list: albums and singles that sound unlike their label",
|
|
13982
|
+
tags: ["Admin"]
|
|
13983
|
+
}).output(object({
|
|
13984
|
+
items: array(LabelOutlierItemSchema),
|
|
13985
|
+
lastRun: LabelOutlierRunSchema.nullable(),
|
|
13986
|
+
ok: literal(true)
|
|
13987
|
+
}));
|
|
13988
|
+
setLabelOutliersDismissed = oc.route({
|
|
13989
|
+
method: "PUT",
|
|
13990
|
+
operationId: "setLabelOutliersDismissed",
|
|
13991
|
+
path: "/admin/label-outliers/dismissed",
|
|
13992
|
+
summary: "Mark flagged label outliers as looking fine, or put them back on the list (operator)",
|
|
13993
|
+
tags: ["Admin"]
|
|
13994
|
+
}).input(object({
|
|
13995
|
+
dismissed: boolean2(),
|
|
13996
|
+
unitIds: array(string2().min(1).max(256)).min(1).max(MAX_LABEL_OUTLIER_DISMISSALS)
|
|
13997
|
+
})).output(object({ changed: number2().int(), ok: literal(true) }));
|
|
13998
|
+
LabelOutlierInputTrackSchema = object({
|
|
13999
|
+
albumId: string2().nullable(),
|
|
14000
|
+
artistIds: array(string2()),
|
|
14001
|
+
embeddingBase64: string2(),
|
|
14002
|
+
labelId: string2().nullable(),
|
|
14003
|
+
trackId: string2()
|
|
14004
|
+
}).meta({ id: "LabelOutlierInputTrack" });
|
|
14005
|
+
LabelOutlierInputAlbumSchema = object({ discogsStyles: string2(), id: string2() }).meta({ id: "LabelOutlierInputAlbum" });
|
|
14006
|
+
listLabelOutlierInputs = oc.route({
|
|
14007
|
+
method: "GET",
|
|
14008
|
+
operationId: "listLabelOutlierInputs",
|
|
14009
|
+
path: "/admin/label-outliers/inputs",
|
|
14010
|
+
summary: "One keyset page of the label-outlier scoring inputs: embedded catalogue tracks with their raw vector bytes, artist credits, and album Discogs styles",
|
|
14011
|
+
tags: ["Admin"]
|
|
14012
|
+
}).input(object({ cursor: string2().max(512).optional(), limit: string2().optional() })).output(object({
|
|
14013
|
+
albums: array(LabelOutlierInputAlbumSchema),
|
|
14014
|
+
nextCursor: string2().nullable(),
|
|
14015
|
+
ok: literal(true),
|
|
14016
|
+
tracks: array(LabelOutlierInputTrackSchema)
|
|
14017
|
+
}));
|
|
14018
|
+
adminLabelOutliersContract = {
|
|
14019
|
+
acknowledge_label_outlier_alerts: acknowledgeLabelOutlierAlerts,
|
|
14020
|
+
list_label_outlier_inputs: listLabelOutlierInputs,
|
|
14021
|
+
list_label_outliers: listLabelOutliers,
|
|
14022
|
+
record_label_outliers: recordLabelOutliers,
|
|
14023
|
+
set_label_outliers_dismissed: setLabelOutliersDismissed
|
|
14024
|
+
};
|
|
14025
|
+
});
|
|
14026
|
+
|
|
13827
14027
|
// ../../packages/contracts/src/orpc/admin-labels.ts
|
|
13828
14028
|
var LabelSeedStateSchema, LabelTriageVerdictSchema, LabelAdminItemSchema, listLabelsAdmin, updateLabel, listLabelArtistRules, LABEL_ARTIST_RULES_MAX = 100, LABEL_TRIAGE_RULE_PROPOSALS_MAX, replaceLabelArtistRules, MergeLabelResultSchema, mergeLabel, MB_LABEL_MBID_PATTERN, MintLabelOutcomeSchema, LabelTakeOverResultSchema, mintLabel, LabelAliasSourceSchema, LabelAliasKindSchema, LabelAliasCandidateSchema, listLabelAliases, confirmLabelAlias, rejectLabelAlias, DescribeLabelBodySchema, describeLabel, draftLabelBio, LabelBioWorkItemSchema, listLabelsMissingBio, TriageRuleProposalSchema, RecordLabelTriageBodySchema, recordLabelTriage, adminLabelsContract;
|
|
13829
14029
|
var init_admin_labels = __esm(() => {
|
|
@@ -15872,6 +16072,145 @@ var init_admin_telemetry2 = __esm(() => {
|
|
|
15872
16072
|
};
|
|
15873
16073
|
});
|
|
15874
16074
|
|
|
16075
|
+
// ../../packages/contracts/src/orpc/admin-turso-usage.ts
|
|
16076
|
+
var MAX_TURSO_USAGE_DATABASES = 500, TURSO_USAGE_THRESHOLD_MIN_USD = 1, TURSO_USAGE_THRESHOLD_MAX_USD = 1e5, Count, CycleKey, TursoUsageResourceKeySchema, TursoUsageRateBasisSchema, TursoUsageTotalsSchema, TursoDatabaseUsageSchema, TursoUsageResourceSchema, TursoAttributedDatabaseSchema, TursoUsageSnapshotSchema, TursoUsageHistoryDaySchema, TursoUsageAlertSchema, TursoUsageAlertKeySchema, recordTursoUsage, getTursoUsage, acknowledgeTursoUsageAlerts, setTursoUsageThreshold, adminTursoUsageContract;
|
|
16077
|
+
var init_admin_turso_usage = __esm(() => {
|
|
16078
|
+
init_dist3();
|
|
16079
|
+
init_zod();
|
|
16080
|
+
Count = number2().int().min(0);
|
|
16081
|
+
CycleKey = string2().regex(/^\d{4}-\d{2}$/);
|
|
16082
|
+
TursoUsageResourceKeySchema = _enum([
|
|
16083
|
+
"rowsRead",
|
|
16084
|
+
"rowsWritten",
|
|
16085
|
+
"storage",
|
|
16086
|
+
"embeddedSyncs"
|
|
16087
|
+
]);
|
|
16088
|
+
TursoUsageRateBasisSchema = _enum(["recent", "cycle-to-date", "none"]);
|
|
16089
|
+
TursoUsageTotalsSchema = object({
|
|
16090
|
+
bytesSynced: Count,
|
|
16091
|
+
rowsRead: Count,
|
|
16092
|
+
rowsWritten: Count,
|
|
16093
|
+
storageBytes: Count
|
|
16094
|
+
}).meta({ id: "TursoUsageTotals" });
|
|
16095
|
+
TursoDatabaseUsageSchema = TursoUsageTotalsSchema.extend({
|
|
16096
|
+
name: string2().min(1).max(256)
|
|
16097
|
+
}).meta({ id: "TursoDatabaseUsage" });
|
|
16098
|
+
TursoUsageResourceSchema = object({
|
|
16099
|
+
dailyRate: number2().min(0),
|
|
16100
|
+
included: number2().min(0),
|
|
16101
|
+
key: TursoUsageResourceKeySchema,
|
|
16102
|
+
overageUsd: number2().min(0),
|
|
16103
|
+
projectedOverageUsd: number2().min(0),
|
|
16104
|
+
projectedUsed: number2().min(0),
|
|
16105
|
+
unitSize: number2().positive(),
|
|
16106
|
+
usdPerUnit: number2().min(0),
|
|
16107
|
+
used: number2().min(0)
|
|
16108
|
+
}).meta({ id: "TursoUsageResource" });
|
|
16109
|
+
TursoAttributedDatabaseSchema = TursoDatabaseUsageSchema.extend({
|
|
16110
|
+
attributedOverageUsd: number2().min(0)
|
|
16111
|
+
}).meta({ id: "TursoAttributedDatabase" });
|
|
16112
|
+
TursoUsageSnapshotSchema = object({
|
|
16113
|
+
baseUsd: number2().min(0).nullable(),
|
|
16114
|
+
cycle: CycleKey,
|
|
16115
|
+
cycleEnd: string2(),
|
|
16116
|
+
cycleStart: string2(),
|
|
16117
|
+
databases: array(TursoAttributedDatabaseSchema),
|
|
16118
|
+
observedAt: string2(),
|
|
16119
|
+
overageUsd: number2().min(0),
|
|
16120
|
+
overagesEnabled: boolean2(),
|
|
16121
|
+
plan: string2(),
|
|
16122
|
+
priceSource: string2(),
|
|
16123
|
+
priceTableVersion: string2(),
|
|
16124
|
+
priced: boolean2(),
|
|
16125
|
+
projectedBillUsd: number2().min(0).nullable(),
|
|
16126
|
+
projectedOverageUsd: number2().min(0),
|
|
16127
|
+
rateBasis: TursoUsageRateBasisSchema,
|
|
16128
|
+
rateWindowHours: number2().min(0).nullable(),
|
|
16129
|
+
resources: array(TursoUsageResourceSchema),
|
|
16130
|
+
timeline: string2().nullable(),
|
|
16131
|
+
upcomingInvoiceUsd: number2().min(0).nullable()
|
|
16132
|
+
}).meta({ id: "TursoUsageSnapshot" });
|
|
16133
|
+
TursoUsageHistoryDaySchema = object({
|
|
16134
|
+
bytesSynced: Count,
|
|
16135
|
+
day: string2(),
|
|
16136
|
+
overageUsd: number2().min(0),
|
|
16137
|
+
projectedOverageUsd: number2().min(0),
|
|
16138
|
+
rowsRead: Count,
|
|
16139
|
+
rowsWritten: Count,
|
|
16140
|
+
storageBytes: Count
|
|
16141
|
+
}).meta({ id: "TursoUsageHistoryDay" });
|
|
16142
|
+
TursoUsageAlertSchema = object({
|
|
16143
|
+
cycle: CycleKey,
|
|
16144
|
+
deliveredAt: string2().nullable(),
|
|
16145
|
+
levelUsd: number2().positive(),
|
|
16146
|
+
projectedOverageUsd: number2().min(0),
|
|
16147
|
+
raisedAt: string2()
|
|
16148
|
+
}).meta({ id: "TursoUsageAlert" });
|
|
16149
|
+
TursoUsageAlertKeySchema = object({
|
|
16150
|
+
cycle: CycleKey,
|
|
16151
|
+
levelUsd: number2().positive()
|
|
16152
|
+
}).meta({ id: "TursoUsageAlertKey" });
|
|
16153
|
+
recordTursoUsage = oc.route({
|
|
16154
|
+
method: "POST",
|
|
16155
|
+
operationId: "recordTursoUsage",
|
|
16156
|
+
path: "/admin/costs/turso-usage",
|
|
16157
|
+
summary: "Record one Turso platform usage reading and price it against the plan",
|
|
16158
|
+
tags: ["Admin"]
|
|
16159
|
+
}).input(object({
|
|
16160
|
+
databases: array(TursoDatabaseUsageSchema).max(MAX_TURSO_USAGE_DATABASES),
|
|
16161
|
+
observedAt: string2().min(1).max(64),
|
|
16162
|
+
plan: object({
|
|
16163
|
+
name: string2().min(1).max(64),
|
|
16164
|
+
overages: boolean2(),
|
|
16165
|
+
timeline: string2().max(32).nullable()
|
|
16166
|
+
}),
|
|
16167
|
+
upcomingInvoiceUsd: number2().min(0).nullable(),
|
|
16168
|
+
usage: TursoUsageTotalsSchema
|
|
16169
|
+
})).output(object({
|
|
16170
|
+
ok: literal(true),
|
|
16171
|
+
pendingAlerts: array(TursoUsageAlertSchema),
|
|
16172
|
+
snapshot: TursoUsageSnapshotSchema,
|
|
16173
|
+
stored: boolean2(),
|
|
16174
|
+
thresholdUsd: number2().positive()
|
|
16175
|
+
}));
|
|
16176
|
+
getTursoUsage = oc.route({
|
|
16177
|
+
method: "GET",
|
|
16178
|
+
operationId: "getTursoUsage",
|
|
16179
|
+
path: "/admin/costs/turso-usage",
|
|
16180
|
+
summary: "The latest priced Turso usage reading, its per-day history, and this cycle's alerts (operator)",
|
|
16181
|
+
tags: ["Admin"]
|
|
16182
|
+
}).output(object({
|
|
16183
|
+
alerts: array(TursoUsageAlertSchema),
|
|
16184
|
+
available: boolean2(),
|
|
16185
|
+
history: array(TursoUsageHistoryDaySchema),
|
|
16186
|
+
latest: TursoUsageSnapshotSchema.nullable(),
|
|
16187
|
+
ok: literal(true),
|
|
16188
|
+
thresholdUsd: number2().positive()
|
|
16189
|
+
}));
|
|
16190
|
+
acknowledgeTursoUsageAlerts = oc.route({
|
|
16191
|
+
method: "PUT",
|
|
16192
|
+
operationId: "acknowledgeTursoUsageAlerts",
|
|
16193
|
+
path: "/admin/costs/turso-usage/alerts",
|
|
16194
|
+
summary: "Mark Turso overage alerts as delivered, once their Discord post has landed",
|
|
16195
|
+
tags: ["Admin"]
|
|
16196
|
+
}).input(object({ alerts: array(TursoUsageAlertKeySchema).min(1).max(20) })).output(object({ acknowledged: number2().int().min(0), ok: literal(true) }));
|
|
16197
|
+
setTursoUsageThreshold = oc.route({
|
|
16198
|
+
method: "PUT",
|
|
16199
|
+
operationId: "setTursoUsageThreshold",
|
|
16200
|
+
path: "/admin/costs/turso-usage/threshold",
|
|
16201
|
+
summary: "Set the projected Turso overage that raises a Discord alert (operator)",
|
|
16202
|
+
tags: ["Admin"]
|
|
16203
|
+
}).input(object({
|
|
16204
|
+
thresholdUsd: number2().min(TURSO_USAGE_THRESHOLD_MIN_USD).max(TURSO_USAGE_THRESHOLD_MAX_USD)
|
|
16205
|
+
})).output(object({ ok: literal(true), thresholdUsd: number2().positive() }));
|
|
16206
|
+
adminTursoUsageContract = {
|
|
16207
|
+
acknowledge_turso_usage_alerts: acknowledgeTursoUsageAlerts,
|
|
16208
|
+
get_turso_usage: getTursoUsage,
|
|
16209
|
+
record_turso_usage: recordTursoUsage,
|
|
16210
|
+
set_turso_usage_threshold: setTursoUsageThreshold
|
|
16211
|
+
};
|
|
16212
|
+
});
|
|
16213
|
+
|
|
15875
16214
|
// ../../packages/contracts/src/orpc/admin-twitch.ts
|
|
15876
16215
|
var recordLiveState, adminTwitchContract;
|
|
15877
16216
|
var init_admin_twitch = __esm(() => {
|
|
@@ -17286,6 +17625,7 @@ var init_orpc = __esm(() => {
|
|
|
17286
17625
|
init_admin_funnel();
|
|
17287
17626
|
init_admin_health();
|
|
17288
17627
|
init_admin_hub_counts();
|
|
17628
|
+
init_admin_label_outliers();
|
|
17289
17629
|
init_admin_labels();
|
|
17290
17630
|
init_admin_logbook();
|
|
17291
17631
|
init_admin_migrations();
|
|
@@ -17300,6 +17640,7 @@ var init_orpc = __esm(() => {
|
|
|
17300
17640
|
init_admin_tokens();
|
|
17301
17641
|
init_admin_tracks();
|
|
17302
17642
|
init_admin_telemetry2();
|
|
17643
|
+
init_admin_turso_usage();
|
|
17303
17644
|
init_admin_twitch();
|
|
17304
17645
|
init_admin_users();
|
|
17305
17646
|
init_devices();
|
|
@@ -17332,6 +17673,7 @@ var init_orpc = __esm(() => {
|
|
|
17332
17673
|
init_mix();
|
|
17333
17674
|
init_galaxies2();
|
|
17334
17675
|
init_graph();
|
|
17676
|
+
init_admin_label_outliers();
|
|
17335
17677
|
init_admin_galaxies();
|
|
17336
17678
|
init_admin_albums();
|
|
17337
17679
|
init_admin_artifacts();
|
|
@@ -17349,6 +17691,7 @@ var init_orpc = __esm(() => {
|
|
|
17349
17691
|
init_admin_notes();
|
|
17350
17692
|
init_admin_observations();
|
|
17351
17693
|
init_admin_costs();
|
|
17694
|
+
init_admin_turso_usage();
|
|
17352
17695
|
init_admin_database_admission();
|
|
17353
17696
|
init_admin_telemetry2();
|
|
17354
17697
|
init_admin_health();
|
|
@@ -17417,6 +17760,7 @@ var init_orpc = __esm(() => {
|
|
|
17417
17760
|
...adminHealthContract,
|
|
17418
17761
|
...adminHubCountsContract,
|
|
17419
17762
|
...followDigestContract,
|
|
17763
|
+
...adminLabelOutliersContract,
|
|
17420
17764
|
...adminLabelsContract,
|
|
17421
17765
|
...adminLogbookContract,
|
|
17422
17766
|
...adminMigrationsContract,
|
|
@@ -17431,6 +17775,7 @@ var init_orpc = __esm(() => {
|
|
|
17431
17775
|
...adminTokensContract,
|
|
17432
17776
|
...adminTracksContract,
|
|
17433
17777
|
...adminTelemetryContract,
|
|
17778
|
+
...adminTursoUsageContract,
|
|
17434
17779
|
...adminTwitchContract,
|
|
17435
17780
|
...adminUsersContract,
|
|
17436
17781
|
...devicesContract,
|
|
@@ -19858,6 +20203,974 @@ var init_admin_covers = __esm(() => {
|
|
|
19858
20203
|
init_api();
|
|
19859
20204
|
});
|
|
19860
20205
|
|
|
20206
|
+
// src/evidence-http.ts
|
|
20207
|
+
import { createHash } from "node:crypto";
|
|
20208
|
+
import { mkdir as mkdir2, readFile as readFile2, rename, rm, stat, writeFile as writeFile2 } from "node:fs/promises";
|
|
20209
|
+
import { homedir as homedir4 } from "node:os";
|
|
20210
|
+
import { join as join6 } from "node:path";
|
|
20211
|
+
function defaultCacheDir() {
|
|
20212
|
+
const base = process.env.XDG_CACHE_HOME || join6(homedir4(), ".cache");
|
|
20213
|
+
return join6(base, "fluncle", "label-evidence");
|
|
20214
|
+
}
|
|
20215
|
+
function createEvidenceHttp(overrides = {}) {
|
|
20216
|
+
return {
|
|
20217
|
+
cacheDir: defaultCacheDir(),
|
|
20218
|
+
cacheTtlMs: 7 * 24 * 60 * 60 * 1000,
|
|
20219
|
+
fetch: (url, init) => globalThis.fetch(url, init),
|
|
20220
|
+
now: () => Date.now(),
|
|
20221
|
+
policies: SOURCE_POLICIES,
|
|
20222
|
+
random: Math.random,
|
|
20223
|
+
refresh: false,
|
|
20224
|
+
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
20225
|
+
stats: { cacheHits: 0, requests: 0 },
|
|
20226
|
+
...overrides
|
|
20227
|
+
};
|
|
20228
|
+
}
|
|
20229
|
+
function cachePath(http, source, key) {
|
|
20230
|
+
const digest = createHash("sha256").update(key).digest("hex");
|
|
20231
|
+
return join6(http.cacheDir, source, `${digest}.json`);
|
|
20232
|
+
}
|
|
20233
|
+
async function readCache2(http, source, key) {
|
|
20234
|
+
if (http.refresh) {
|
|
20235
|
+
return null;
|
|
20236
|
+
}
|
|
20237
|
+
try {
|
|
20238
|
+
const entry = JSON.parse(await readFile2(cachePath(http, source, key), "utf8"));
|
|
20239
|
+
if (entry.key !== key || http.now() - entry.fetchedAt > http.cacheTtlMs) {
|
|
20240
|
+
return null;
|
|
20241
|
+
}
|
|
20242
|
+
return entry.text;
|
|
20243
|
+
} catch {
|
|
20244
|
+
return null;
|
|
20245
|
+
}
|
|
20246
|
+
}
|
|
20247
|
+
async function writeCache2(http, source, key, text) {
|
|
20248
|
+
const path = cachePath(http, source, key);
|
|
20249
|
+
const temp = `${path}.${process.pid}.${Math.floor(http.random() * 1e9)}.tmp`;
|
|
20250
|
+
try {
|
|
20251
|
+
await mkdir2(join6(http.cacheDir, source), { recursive: true });
|
|
20252
|
+
await writeFile2(temp, JSON.stringify({ fetchedAt: http.now(), key, text }));
|
|
20253
|
+
await rename(temp, path);
|
|
20254
|
+
} catch {
|
|
20255
|
+
await rm(temp, { force: true }).catch(() => {
|
|
20256
|
+
return;
|
|
20257
|
+
});
|
|
20258
|
+
}
|
|
20259
|
+
}
|
|
20260
|
+
async function withSlotLock(http, source, critical) {
|
|
20261
|
+
const dir = join6(http.cacheDir, "slots");
|
|
20262
|
+
await mkdir2(dir, { recursive: true });
|
|
20263
|
+
const lock = join6(dir, `${source}.lock`);
|
|
20264
|
+
for (;; ) {
|
|
20265
|
+
try {
|
|
20266
|
+
await mkdir2(lock);
|
|
20267
|
+
break;
|
|
20268
|
+
} catch (error) {
|
|
20269
|
+
if (error.code !== "EEXIST") {
|
|
20270
|
+
throw error;
|
|
20271
|
+
}
|
|
20272
|
+
const held = await stat(lock).catch(() => null);
|
|
20273
|
+
if (held && Date.now() - held.mtimeMs > LOCK_STALE_MS) {
|
|
20274
|
+
const evicted = `${lock}.stale.${process.pid}.${Math.floor(Math.random() * 1e9)}`;
|
|
20275
|
+
const won = await rename(lock, evicted).then(() => true, () => false);
|
|
20276
|
+
if (won) {
|
|
20277
|
+
await rm(evicted, { force: true, recursive: true });
|
|
20278
|
+
}
|
|
20279
|
+
continue;
|
|
20280
|
+
}
|
|
20281
|
+
await new Promise((resolve) => setTimeout(resolve, 15 + Math.floor(Math.random() * 35)));
|
|
20282
|
+
}
|
|
20283
|
+
}
|
|
20284
|
+
try {
|
|
20285
|
+
return await critical(join6(dir, `${source}.next`));
|
|
20286
|
+
} finally {
|
|
20287
|
+
await rm(lock, { force: true, recursive: true });
|
|
20288
|
+
}
|
|
20289
|
+
}
|
|
20290
|
+
async function readNextSlot(slotFile) {
|
|
20291
|
+
const raw = await readFile2(slotFile, "utf8").catch(() => "0");
|
|
20292
|
+
const value = Number(raw);
|
|
20293
|
+
return Number.isFinite(value) ? value : 0;
|
|
20294
|
+
}
|
|
20295
|
+
async function reserveSlot(http, source) {
|
|
20296
|
+
const interval = http.policies[source].intervalMs;
|
|
20297
|
+
const slotAt = await withSlotLock(http, source, async (slotFile) => {
|
|
20298
|
+
const now = http.now();
|
|
20299
|
+
const slot = Math.max(now, await readNextSlot(slotFile));
|
|
20300
|
+
await writeFile2(slotFile, String(slot + interval));
|
|
20301
|
+
return slot;
|
|
20302
|
+
});
|
|
20303
|
+
const wait = slotAt - http.now();
|
|
20304
|
+
if (wait > 0) {
|
|
20305
|
+
await http.sleep(wait);
|
|
20306
|
+
}
|
|
20307
|
+
}
|
|
20308
|
+
async function pushSlotBack(http, source, until) {
|
|
20309
|
+
await withSlotLock(http, source, async (slotFile) => {
|
|
20310
|
+
if (await readNextSlot(slotFile) < until) {
|
|
20311
|
+
await writeFile2(slotFile, String(until));
|
|
20312
|
+
}
|
|
20313
|
+
});
|
|
20314
|
+
}
|
|
20315
|
+
function parseRetryAfter(value, now) {
|
|
20316
|
+
if (!value) {
|
|
20317
|
+
return null;
|
|
20318
|
+
}
|
|
20319
|
+
const seconds = Number(value);
|
|
20320
|
+
if (Number.isFinite(seconds)) {
|
|
20321
|
+
return Math.max(0, seconds * 1000);
|
|
20322
|
+
}
|
|
20323
|
+
const at = Date.parse(value);
|
|
20324
|
+
return Number.isNaN(at) ? null : Math.max(0, at - now);
|
|
20325
|
+
}
|
|
20326
|
+
function isRetryableStatus(status) {
|
|
20327
|
+
return status === 429 || status >= 500 && status !== 501;
|
|
20328
|
+
}
|
|
20329
|
+
function failureKind(status) {
|
|
20330
|
+
if (status === 404) {
|
|
20331
|
+
return "not_found";
|
|
20332
|
+
}
|
|
20333
|
+
return status === 429 || status === 503 ? "rate_limited" : "http";
|
|
20334
|
+
}
|
|
20335
|
+
async function performWithTimeout(perform, timeoutMs) {
|
|
20336
|
+
const controller = new AbortController;
|
|
20337
|
+
let timer;
|
|
20338
|
+
const timeout = new Promise((resolve) => {
|
|
20339
|
+
timer = setTimeout(() => {
|
|
20340
|
+
controller.abort();
|
|
20341
|
+
resolve({ timedOut: true });
|
|
20342
|
+
}, timeoutMs);
|
|
20343
|
+
});
|
|
20344
|
+
try {
|
|
20345
|
+
return await Promise.race([
|
|
20346
|
+
perform(controller.signal).then((response) => ({ response }), (error) => controller.signal.aborted ? { timedOut: true } : { error }),
|
|
20347
|
+
timeout
|
|
20348
|
+
]);
|
|
20349
|
+
} finally {
|
|
20350
|
+
clearTimeout(timer);
|
|
20351
|
+
}
|
|
20352
|
+
}
|
|
20353
|
+
async function fetchEvidenceText(http, source, cacheKey, perform, accept = () => true) {
|
|
20354
|
+
const hit = await readCache2(http, source, cacheKey);
|
|
20355
|
+
if (hit !== null && accept(hit)) {
|
|
20356
|
+
http.stats.cacheHits += 1;
|
|
20357
|
+
return { cached: true, text: hit };
|
|
20358
|
+
}
|
|
20359
|
+
const policy = http.policies[source];
|
|
20360
|
+
let last;
|
|
20361
|
+
for (let attempt = 1;attempt <= policy.attempts; attempt += 1) {
|
|
20362
|
+
await reserveSlot(http, source);
|
|
20363
|
+
http.stats.requests += 1;
|
|
20364
|
+
const outcome = await performWithTimeout(perform, policy.timeoutMs);
|
|
20365
|
+
let retryAfterMs = null;
|
|
20366
|
+
if ("response" in outcome) {
|
|
20367
|
+
const { response } = outcome;
|
|
20368
|
+
const success = response.status >= 200 && response.status < 300;
|
|
20369
|
+
if (success && accept(response.text)) {
|
|
20370
|
+
await writeCache2(http, source, cacheKey, response.text);
|
|
20371
|
+
return { cached: false, text: response.text };
|
|
20372
|
+
}
|
|
20373
|
+
if (success) {
|
|
20374
|
+
last = new EvidenceFetchError("invalid", `${source} answered HTTP ${response.status} with a body that is not the expected data`, attempt, response.status);
|
|
20375
|
+
} else {
|
|
20376
|
+
last = new EvidenceFetchError(failureKind(response.status), `${source} answered HTTP ${response.status}`, attempt, response.status);
|
|
20377
|
+
if (!isRetryableStatus(response.status)) {
|
|
20378
|
+
throw last;
|
|
20379
|
+
}
|
|
20380
|
+
retryAfterMs = parseRetryAfter(response.headers.get("retry-after"), http.now());
|
|
20381
|
+
if (retryAfterMs !== null) {
|
|
20382
|
+
await pushSlotBack(http, source, http.now() + Math.min(retryAfterMs, policy.maxBackoffMs));
|
|
20383
|
+
}
|
|
20384
|
+
if (retryAfterMs !== null && retryAfterMs > policy.maxBackoffMs) {
|
|
20385
|
+
throw new EvidenceFetchError("rate_limited", `${source} asked to wait ${Math.ceil(retryAfterMs / 1000)} s, past the ${policy.maxBackoffMs / 1000} s limit`, attempt, response.status);
|
|
20386
|
+
}
|
|
20387
|
+
}
|
|
20388
|
+
} else if ("timedOut" in outcome) {
|
|
20389
|
+
last = new EvidenceFetchError("timeout", `${source} did not answer within ${policy.timeoutMs} ms`, attempt);
|
|
20390
|
+
} else {
|
|
20391
|
+
const reason = outcome.error instanceof Error ? outcome.error.message : String(outcome.error);
|
|
20392
|
+
last = new EvidenceFetchError("network", `${source} request failed: ${reason}`, attempt);
|
|
20393
|
+
}
|
|
20394
|
+
if (attempt === policy.attempts) {
|
|
20395
|
+
break;
|
|
20396
|
+
}
|
|
20397
|
+
const exponential = policy.intervalMs * 2 ** attempt;
|
|
20398
|
+
const backoff = Math.min(policy.maxBackoffMs, Math.max(retryAfterMs ?? 0, exponential) + Math.floor(http.random() * 250));
|
|
20399
|
+
await http.sleep(backoff);
|
|
20400
|
+
}
|
|
20401
|
+
throw new EvidenceFetchError(last?.kind ?? "network", `${last?.message ?? `${source} request failed`} (gave up after ${policy.attempts} attempts)`, policy.attempts, last?.status);
|
|
20402
|
+
}
|
|
20403
|
+
function isJson(text) {
|
|
20404
|
+
try {
|
|
20405
|
+
JSON.parse(text);
|
|
20406
|
+
return true;
|
|
20407
|
+
} catch {
|
|
20408
|
+
return false;
|
|
20409
|
+
}
|
|
20410
|
+
}
|
|
20411
|
+
async function fetchEvidenceJson(http, source, url, headers = {}) {
|
|
20412
|
+
const { cached, text } = await fetchEvidenceText(http, source, `GET ${url}`, async (signal) => {
|
|
20413
|
+
const response = await http.fetch(url, {
|
|
20414
|
+
headers: { Accept: "application/json", "User-Agent": EVIDENCE_USER_AGENT, ...headers },
|
|
20415
|
+
signal
|
|
20416
|
+
});
|
|
20417
|
+
return { headers: response.headers, status: response.status, text: await response.text() };
|
|
20418
|
+
}, isJson);
|
|
20419
|
+
return { cached, data: JSON.parse(text) };
|
|
20420
|
+
}
|
|
20421
|
+
var SOURCE_POLICIES, EVIDENCE_USER_AGENT = "FluncleLabelTriage/1.0 ( https://www.fluncle.com )", EvidenceFetchError, LOCK_STALE_MS = 1e4;
|
|
20422
|
+
var init_evidence_http = __esm(() => {
|
|
20423
|
+
SOURCE_POLICIES = {
|
|
20424
|
+
apple: { attempts: 3, intervalMs: 3000, maxBackoffMs: 30000, timeoutMs: 1e4 },
|
|
20425
|
+
beatport: { attempts: 2, intervalMs: 1000, maxBackoffMs: 30000, timeoutMs: 75000 },
|
|
20426
|
+
discogs: { attempts: 3, intervalMs: 2500, maxBackoffMs: 30000, timeoutMs: 15000 },
|
|
20427
|
+
musicbrainz: { attempts: 4, intervalMs: 1100, maxBackoffMs: 30000, timeoutMs: 20000 }
|
|
20428
|
+
};
|
|
20429
|
+
EvidenceFetchError = class EvidenceFetchError extends Error {
|
|
20430
|
+
attempts;
|
|
20431
|
+
kind;
|
|
20432
|
+
status;
|
|
20433
|
+
constructor(kind, message, attempts, status) {
|
|
20434
|
+
super(message);
|
|
20435
|
+
this.name = "EvidenceFetchError";
|
|
20436
|
+
this.kind = kind;
|
|
20437
|
+
this.attempts = attempts;
|
|
20438
|
+
this.status = status;
|
|
20439
|
+
}
|
|
20440
|
+
};
|
|
20441
|
+
});
|
|
20442
|
+
|
|
20443
|
+
// src/commands/admin-labels.ts
|
|
20444
|
+
var exports_admin_labels = {};
|
|
20445
|
+
__export(exports_admin_labels, {
|
|
20446
|
+
resolveLabel: () => resolveLabel2
|
|
20447
|
+
});
|
|
20448
|
+
async function resolveLabel2(slugOrId) {
|
|
20449
|
+
const { labels } = await adminApiGet("/api/v1/admin/labels");
|
|
20450
|
+
const match = labels.find((label) => label.slug === slugOrId || label.id === slugOrId);
|
|
20451
|
+
if (!match) {
|
|
20452
|
+
throw new Error(`No label with slug or id '${slugOrId}' — check \`fluncle labels\``);
|
|
20453
|
+
}
|
|
20454
|
+
return match;
|
|
20455
|
+
}
|
|
20456
|
+
var init_admin_labels3 = __esm(() => {
|
|
20457
|
+
init_api();
|
|
20458
|
+
init_admin_artists3();
|
|
20459
|
+
});
|
|
20460
|
+
|
|
20461
|
+
// src/commands/admin-label-evidence.ts
|
|
20462
|
+
import { spawn } from "node:child_process";
|
|
20463
|
+
function parseCensusPages(value) {
|
|
20464
|
+
const pages = Number(value);
|
|
20465
|
+
if (!Number.isInteger(pages) || pages < 1 || pages > MAX_CENSUS_PAGES) {
|
|
20466
|
+
throw new Error(`Pass --census-pages 1-${MAX_CENSUS_PAGES}`);
|
|
20467
|
+
}
|
|
20468
|
+
return pages;
|
|
20469
|
+
}
|
|
20470
|
+
function parseEvidenceSources(value) {
|
|
20471
|
+
if (value === undefined || value.trim() === "") {
|
|
20472
|
+
return [...EVIDENCE_SOURCES];
|
|
20473
|
+
}
|
|
20474
|
+
const wanted = value.split(",").map((entry) => entry.trim().toLowerCase());
|
|
20475
|
+
const unknown = wanted.filter((entry) => !EVIDENCE_SOURCES.includes(entry));
|
|
20476
|
+
if (unknown.length > 0) {
|
|
20477
|
+
throw new Error(`Unknown source(s): ${unknown.join(", ")} — pick from ${EVIDENCE_SOURCES.join(", ")}`);
|
|
20478
|
+
}
|
|
20479
|
+
return EVIDENCE_SOURCES.filter((source) => wanted.includes(source));
|
|
20480
|
+
}
|
|
20481
|
+
function failureOf(error, request) {
|
|
20482
|
+
if (error instanceof EvidenceFetchError) {
|
|
20483
|
+
return {
|
|
20484
|
+
attempts: error.attempts,
|
|
20485
|
+
kind: error.kind,
|
|
20486
|
+
message: error.message,
|
|
20487
|
+
request,
|
|
20488
|
+
...error.status === undefined ? {} : { status: error.status }
|
|
20489
|
+
};
|
|
20490
|
+
}
|
|
20491
|
+
return {
|
|
20492
|
+
kind: "parse",
|
|
20493
|
+
message: error instanceof Error ? error.message : String(error),
|
|
20494
|
+
request
|
|
20495
|
+
};
|
|
20496
|
+
}
|
|
20497
|
+
function statusFrom(parts, failures) {
|
|
20498
|
+
if (failures === 0) {
|
|
20499
|
+
return "ok";
|
|
20500
|
+
}
|
|
20501
|
+
return failures < parts ? "partial" : "error";
|
|
20502
|
+
}
|
|
20503
|
+
function tally(names) {
|
|
20504
|
+
const counts = new Map;
|
|
20505
|
+
for (const name of names) {
|
|
20506
|
+
counts.set(name, (counts.get(name) ?? 0) + 1);
|
|
20507
|
+
}
|
|
20508
|
+
return [...counts.entries()].map(([name, count]) => ({ count, name })).sort((a, b) => b.count - a.count || a.name.localeCompare(b.name));
|
|
20509
|
+
}
|
|
20510
|
+
function firstCredit(credit) {
|
|
20511
|
+
const head = credit?.[0];
|
|
20512
|
+
const mbid = head?.artist?.id;
|
|
20513
|
+
if (!mbid) {
|
|
20514
|
+
return null;
|
|
20515
|
+
}
|
|
20516
|
+
return { mbid, name: head?.artist?.name ?? head?.name ?? mbid };
|
|
20517
|
+
}
|
|
20518
|
+
function crawlRuleCredit(credit) {
|
|
20519
|
+
for (const entry of credit ?? []) {
|
|
20520
|
+
const name = entry.artist?.name ?? entry.name;
|
|
20521
|
+
const mbid = entry.artist?.id;
|
|
20522
|
+
if (name && mbid && mbid !== VARIOUS_ARTISTS_MBID) {
|
|
20523
|
+
return { mbid, name };
|
|
20524
|
+
}
|
|
20525
|
+
}
|
|
20526
|
+
return null;
|
|
20527
|
+
}
|
|
20528
|
+
function creditString(credit) {
|
|
20529
|
+
return (credit ?? []).map((entry) => entry.name ?? entry.artist?.name ?? "").join(" / ");
|
|
20530
|
+
}
|
|
20531
|
+
function linkOf(links, host) {
|
|
20532
|
+
return links.find((link) => link.url.includes(host))?.url ?? null;
|
|
20533
|
+
}
|
|
20534
|
+
function toLabelFacts(payload, mbid) {
|
|
20535
|
+
const relations = payload.relations ?? [];
|
|
20536
|
+
return {
|
|
20537
|
+
area: payload.area?.name ?? null,
|
|
20538
|
+
country: payload.country ?? null,
|
|
20539
|
+
disambiguation: payload.disambiguation || null,
|
|
20540
|
+
genres: (payload.genres ?? []).flatMap((genre) => genre.name ? [genre.name] : []),
|
|
20541
|
+
labelCode: payload["label-code"] ?? null,
|
|
20542
|
+
labelRelations: relations.flatMap((relation) => relation.label?.id ? [
|
|
20543
|
+
{
|
|
20544
|
+
direction: relation.direction ?? "forward",
|
|
20545
|
+
mbid: relation.label.id,
|
|
20546
|
+
name: relation.label.name ?? relation.label.id,
|
|
20547
|
+
type: relation.type ?? "unknown"
|
|
20548
|
+
}
|
|
20549
|
+
] : []),
|
|
20550
|
+
lifeSpan: {
|
|
20551
|
+
begin: payload["life-span"]?.begin ?? null,
|
|
20552
|
+
end: payload["life-span"]?.end ?? null,
|
|
20553
|
+
ended: payload["life-span"]?.ended === true
|
|
20554
|
+
},
|
|
20555
|
+
links: relations.flatMap((relation) => relation.url?.resource ? [{ type: relation.type ?? "unknown", url: relation.url.resource }] : []),
|
|
20556
|
+
mbid: payload.id ?? mbid,
|
|
20557
|
+
name: payload.name ?? mbid,
|
|
20558
|
+
tags: (payload.tags ?? []).flatMap((tag) => tag.name ? [tag.name] : []),
|
|
20559
|
+
type: payload.type ?? null
|
|
20560
|
+
};
|
|
20561
|
+
}
|
|
20562
|
+
function toReleaseFacts(browse, mbid) {
|
|
20563
|
+
const releases = browse.releases ?? [];
|
|
20564
|
+
const byArtist = new Map;
|
|
20565
|
+
let variousArtistsReleases = 0;
|
|
20566
|
+
const samples = releases.map((release) => {
|
|
20567
|
+
const head = firstCredit(release["artist-credit"]);
|
|
20568
|
+
if (head?.mbid === VARIOUS_ARTISTS_MBID) {
|
|
20569
|
+
variousArtistsReleases += 1;
|
|
20570
|
+
} else if (head) {
|
|
20571
|
+
const entry = byArtist.get(head.mbid) ?? { mbid: head.mbid, name: head.name, releases: 0 };
|
|
20572
|
+
entry.releases += 1;
|
|
20573
|
+
byArtist.set(head.mbid, entry);
|
|
20574
|
+
}
|
|
20575
|
+
const info = release["label-info"]?.find((entry) => entry.label?.id === mbid);
|
|
20576
|
+
const formats = [
|
|
20577
|
+
...new Set((release.media ?? []).flatMap((m) => m.format ? [m.format] : []))
|
|
20578
|
+
];
|
|
20579
|
+
return {
|
|
20580
|
+
artist: creditString(release["artist-credit"]),
|
|
20581
|
+
barcode: release.barcode || null,
|
|
20582
|
+
catno: info?.["catalog-number"] ?? null,
|
|
20583
|
+
date: release.date || null,
|
|
20584
|
+
format: formats.length > 0 ? formats.join(" + ") : null,
|
|
20585
|
+
mbid: release.id,
|
|
20586
|
+
status: release.status ?? null,
|
|
20587
|
+
title: release.title ?? ""
|
|
20588
|
+
};
|
|
20589
|
+
});
|
|
20590
|
+
return {
|
|
20591
|
+
firstCreditArtists: [...byArtist.values()].sort((a, b) => b.releases - a.releases || a.name.localeCompare(b.name)).slice(0, FIRST_CREDIT_ARTISTS),
|
|
20592
|
+
releaseCount: browse["release-count"] ?? releases.length,
|
|
20593
|
+
releases: samples.sort((a, b) => (b.date ?? "").localeCompare(a.date ?? "")).slice(0, RELEASE_SAMPLE),
|
|
20594
|
+
releasesRead: releases.length,
|
|
20595
|
+
variousArtistsReleases
|
|
20596
|
+
};
|
|
20597
|
+
}
|
|
20598
|
+
async function runCensus(deps, mbid, maxPages) {
|
|
20599
|
+
const credits = new Map;
|
|
20600
|
+
const seen = new Set;
|
|
20601
|
+
let offset = 0;
|
|
20602
|
+
let pagesFetched = 0;
|
|
20603
|
+
let releaseCount = 0;
|
|
20604
|
+
let allCached = true;
|
|
20605
|
+
let failure;
|
|
20606
|
+
let uncredited = 0;
|
|
20607
|
+
while (pagesFetched < maxPages) {
|
|
20608
|
+
const url = `${MB_ROOT}/release?label=${mbid}&inc=artist-credits+recordings&limit=100&offset=${offset}&fmt=json`;
|
|
20609
|
+
let page;
|
|
20610
|
+
try {
|
|
20611
|
+
page = await fetchEvidenceJson(deps.http, "musicbrainz", url);
|
|
20612
|
+
} catch (error) {
|
|
20613
|
+
if (pagesFetched === 0) {
|
|
20614
|
+
throw error;
|
|
20615
|
+
}
|
|
20616
|
+
failure = failureOf(error, url);
|
|
20617
|
+
break;
|
|
20618
|
+
}
|
|
20619
|
+
const { cached, data } = page;
|
|
20620
|
+
const releases = data.releases ?? [];
|
|
20621
|
+
allCached &&= cached;
|
|
20622
|
+
pagesFetched += 1;
|
|
20623
|
+
releaseCount = data["release-count"] ?? releaseCount;
|
|
20624
|
+
for (const release of releases) {
|
|
20625
|
+
for (const medium of release.media ?? []) {
|
|
20626
|
+
for (const track of medium.tracks ?? []) {
|
|
20627
|
+
const recordingId = track.recording?.id;
|
|
20628
|
+
if (!recordingId || seen.has(recordingId)) {
|
|
20629
|
+
continue;
|
|
20630
|
+
}
|
|
20631
|
+
seen.add(recordingId);
|
|
20632
|
+
const head = crawlRuleCredit(track.recording?.["artist-credit"] ?? release["artist-credit"]);
|
|
20633
|
+
if (!head) {
|
|
20634
|
+
uncredited += 1;
|
|
20635
|
+
continue;
|
|
20636
|
+
}
|
|
20637
|
+
const entry = credits.get(head.mbid) ?? {
|
|
20638
|
+
artistMbid: head.mbid,
|
|
20639
|
+
artistName: head.name,
|
|
20640
|
+
recordings: 0
|
|
20641
|
+
};
|
|
20642
|
+
entry.recordings += 1;
|
|
20643
|
+
credits.set(head.mbid, entry);
|
|
20644
|
+
}
|
|
20645
|
+
}
|
|
20646
|
+
}
|
|
20647
|
+
offset += releases.length;
|
|
20648
|
+
if (releases.length === 0 || offset >= releaseCount) {
|
|
20649
|
+
break;
|
|
20650
|
+
}
|
|
20651
|
+
}
|
|
20652
|
+
const sampled = offset < releaseCount;
|
|
20653
|
+
const firstCredits = [...credits.values()].sort((a, b) => b.recordings - a.recordings || a.artistName.localeCompare(b.artistName));
|
|
20654
|
+
return {
|
|
20655
|
+
cached: allCached,
|
|
20656
|
+
...failure ? { failure } : {},
|
|
20657
|
+
census: {
|
|
20658
|
+
caveat: sampled ? `sampled: first ${offset} of ${releaseCount} releases${failure ? " (a later page failed)" : ""}` : null,
|
|
20659
|
+
firstCredits,
|
|
20660
|
+
pagesFetched,
|
|
20661
|
+
recordingsCounted: firstCredits.reduce((sum, entry) => sum + entry.recordings, uncredited),
|
|
20662
|
+
releaseCount,
|
|
20663
|
+
releasesRead: offset,
|
|
20664
|
+
sampled,
|
|
20665
|
+
uncreditedRecordings: uncredited
|
|
20666
|
+
}
|
|
20667
|
+
};
|
|
20668
|
+
}
|
|
20669
|
+
async function gatherMusicBrainz(deps, mbid, options) {
|
|
20670
|
+
const data = {};
|
|
20671
|
+
const errors = [];
|
|
20672
|
+
let parts = 2;
|
|
20673
|
+
let allCached = true;
|
|
20674
|
+
const labelUrl = `${MB_ROOT}/label/${mbid}?inc=aliases+label-rels+url-rels+tags+genres&fmt=json`;
|
|
20675
|
+
try {
|
|
20676
|
+
const { cached, data: payload } = await fetchEvidenceJson(deps.http, "musicbrainz", labelUrl);
|
|
20677
|
+
allCached &&= cached;
|
|
20678
|
+
data.label = toLabelFacts(payload, mbid);
|
|
20679
|
+
} catch (error) {
|
|
20680
|
+
errors.push(failureOf(error, labelUrl));
|
|
20681
|
+
}
|
|
20682
|
+
const releasesUrl = `${MB_ROOT}/release?label=${mbid}&inc=artist-credits+labels+media&limit=100&fmt=json`;
|
|
20683
|
+
try {
|
|
20684
|
+
const { cached, data: browse } = await fetchEvidenceJson(deps.http, "musicbrainz", releasesUrl);
|
|
20685
|
+
allCached &&= cached;
|
|
20686
|
+
data.releases = toReleaseFacts(browse, mbid);
|
|
20687
|
+
} catch (error) {
|
|
20688
|
+
errors.push(failureOf(error, releasesUrl));
|
|
20689
|
+
}
|
|
20690
|
+
if (options.census) {
|
|
20691
|
+
parts += 1;
|
|
20692
|
+
try {
|
|
20693
|
+
const { cached, census, failure } = await runCensus(deps, mbid, options.censusPages);
|
|
20694
|
+
allCached &&= cached;
|
|
20695
|
+
data.census = census;
|
|
20696
|
+
if (failure) {
|
|
20697
|
+
errors.push(failure);
|
|
20698
|
+
}
|
|
20699
|
+
} catch (error) {
|
|
20700
|
+
errors.push(failureOf(error, `${MB_ROOT}/release?label=${mbid}&inc=recordings (census)`));
|
|
20701
|
+
}
|
|
20702
|
+
}
|
|
20703
|
+
return {
|
|
20704
|
+
cached: allCached,
|
|
20705
|
+
data,
|
|
20706
|
+
...errors.length > 0 ? { errors } : {},
|
|
20707
|
+
status: statusFrom(parts, errors.length)
|
|
20708
|
+
};
|
|
20709
|
+
}
|
|
20710
|
+
function discogsLabelIdFrom(url) {
|
|
20711
|
+
const match = url ? /discogs\.com\/(?:[a-z]{2}\/)?label\/(\d+)/i.exec(url) : null;
|
|
20712
|
+
return match?.[1] ? Number(match[1]) : null;
|
|
20713
|
+
}
|
|
20714
|
+
function beatportLabelUrlFrom(url) {
|
|
20715
|
+
const match = url ? /beatport\.com\/label\/([^/?#]+)\/(\d+)/i.exec(url) : null;
|
|
20716
|
+
return match?.[1] && match[2] ? `https://www.beatport.com/label/${match[1]}/${match[2]}` : null;
|
|
20717
|
+
}
|
|
20718
|
+
function discogsHeaders(deps) {
|
|
20719
|
+
return deps.discogsToken ? { Authorization: `Discogs token=${deps.discogsToken}` } : {};
|
|
20720
|
+
}
|
|
20721
|
+
async function discogsCandidates(deps, name) {
|
|
20722
|
+
const searchUrl = `${DISCOGS_ROOT}/database/search?type=label&q=${encodeURIComponent(name)}&per_page=5`;
|
|
20723
|
+
try {
|
|
20724
|
+
const { cached, data } = await fetchEvidenceJson(deps.http, "discogs", searchUrl, discogsHeaders(deps));
|
|
20725
|
+
const candidates = (data.results ?? []).flatMap((result) => result.id ? [
|
|
20726
|
+
{
|
|
20727
|
+
id: result.id,
|
|
20728
|
+
title: result.title ?? "",
|
|
20729
|
+
url: `https://www.discogs.com${result.uri ?? `/label/${result.id}`}`
|
|
20730
|
+
}
|
|
20731
|
+
] : []);
|
|
20732
|
+
return {
|
|
20733
|
+
cached,
|
|
20734
|
+
data: { candidates },
|
|
20735
|
+
note: "The MusicBrainz label has no Discogs link. Candidates are unverified name matches: confirm the identity before citing any of them.",
|
|
20736
|
+
status: "no_link"
|
|
20737
|
+
};
|
|
20738
|
+
} catch (error) {
|
|
20739
|
+
return {
|
|
20740
|
+
errors: [failureOf(error, searchUrl)],
|
|
20741
|
+
note: "The MusicBrainz label has no Discogs link.",
|
|
20742
|
+
status: "no_link"
|
|
20743
|
+
};
|
|
20744
|
+
}
|
|
20745
|
+
}
|
|
20746
|
+
function toDiscogsLabel(payload, labelId, fallbackName) {
|
|
20747
|
+
const sublabels = (payload.sublabels ?? []).flatMap((entry) => entry.name ? [entry.name] : []);
|
|
20748
|
+
const parent = payload.parent_label;
|
|
20749
|
+
return {
|
|
20750
|
+
id: payload.id ?? labelId,
|
|
20751
|
+
name: payload.name ?? fallbackName,
|
|
20752
|
+
parentLabel: parent?.id === undefined ? null : { id: parent.id, name: parent.name ?? "" },
|
|
20753
|
+
profile: (payload.profile ?? "").slice(0, PROFILE_CHARS),
|
|
20754
|
+
sublabelCount: sublabels.length,
|
|
20755
|
+
sublabels: sublabels.slice(0, 30),
|
|
20756
|
+
url: payload.uri ?? `https://www.discogs.com/label/${labelId}`,
|
|
20757
|
+
urls: payload.urls ?? []
|
|
20758
|
+
};
|
|
20759
|
+
}
|
|
20760
|
+
function toDiscogsReleases(payload) {
|
|
20761
|
+
const releases = payload.releases ?? [];
|
|
20762
|
+
const ids = releases.flatMap((release) => {
|
|
20763
|
+
const id = release.type === "master" ? release.main_release : release.id;
|
|
20764
|
+
return typeof id === "number" ? [id] : [];
|
|
20765
|
+
});
|
|
20766
|
+
return {
|
|
20767
|
+
facts: {
|
|
20768
|
+
releaseCount: payload.pagination?.items ?? releases.length,
|
|
20769
|
+
sample: releases.slice(0, DISCOGS_RELEASE_SAMPLE).map((release) => ({
|
|
20770
|
+
artist: release.artist ?? "",
|
|
20771
|
+
catno: release.catno ?? null,
|
|
20772
|
+
format: release.format ?? null,
|
|
20773
|
+
title: release.title ?? "",
|
|
20774
|
+
year: release.year || null
|
|
20775
|
+
})),
|
|
20776
|
+
topArtists: tally(releases.map((release) => release.artist ?? "")).filter((entry) => entry.name !== "").slice(0, 15).map((entry) => ({ name: entry.name, releases: entry.count }))
|
|
20777
|
+
},
|
|
20778
|
+
ids
|
|
20779
|
+
};
|
|
20780
|
+
}
|
|
20781
|
+
async function gatherDiscogs(deps, mb) {
|
|
20782
|
+
const labelFacts = mb.label;
|
|
20783
|
+
if (!labelFacts) {
|
|
20784
|
+
return {
|
|
20785
|
+
note: "MusicBrainz label unavailable, so no Discogs link to follow",
|
|
20786
|
+
status: "skipped"
|
|
20787
|
+
};
|
|
20788
|
+
}
|
|
20789
|
+
const labelId = discogsLabelIdFrom(linkOf(labelFacts.links, "discogs.com"));
|
|
20790
|
+
if (labelId === null) {
|
|
20791
|
+
return discogsCandidates(deps, labelFacts.name);
|
|
20792
|
+
}
|
|
20793
|
+
const headers = discogsHeaders(deps);
|
|
20794
|
+
const data = {};
|
|
20795
|
+
const errors = [];
|
|
20796
|
+
const releaseIds = new Set;
|
|
20797
|
+
let allCached = true;
|
|
20798
|
+
const labelUrl = `${DISCOGS_ROOT}/labels/${labelId}`;
|
|
20799
|
+
try {
|
|
20800
|
+
const { cached, data: payload } = await fetchEvidenceJson(deps.http, "discogs", labelUrl, headers);
|
|
20801
|
+
allCached &&= cached;
|
|
20802
|
+
data.label = toDiscogsLabel(payload, labelId, labelFacts.name);
|
|
20803
|
+
} catch (error) {
|
|
20804
|
+
errors.push(failureOf(error, labelUrl));
|
|
20805
|
+
}
|
|
20806
|
+
const releasesUrl = `${DISCOGS_ROOT}/labels/${labelId}/releases?per_page=100&page=1`;
|
|
20807
|
+
try {
|
|
20808
|
+
const { cached, data: payload } = await fetchEvidenceJson(deps.http, "discogs", releasesUrl, headers);
|
|
20809
|
+
allCached &&= cached;
|
|
20810
|
+
const { facts, ids } = toDiscogsReleases(payload);
|
|
20811
|
+
data.releases = facts;
|
|
20812
|
+
for (const id of ids) {
|
|
20813
|
+
releaseIds.add(id);
|
|
20814
|
+
}
|
|
20815
|
+
} catch (error) {
|
|
20816
|
+
errors.push(failureOf(error, releasesUrl));
|
|
20817
|
+
}
|
|
20818
|
+
if (releaseIds.size > 0) {
|
|
20819
|
+
const styled = await readDiscogsStyles(deps, releaseIds, data.label?.name ?? labelFacts.name);
|
|
20820
|
+
allCached &&= styled.cached;
|
|
20821
|
+
errors.push(...styled.errors);
|
|
20822
|
+
data.styles = styled.styles;
|
|
20823
|
+
}
|
|
20824
|
+
return {
|
|
20825
|
+
cached: allCached,
|
|
20826
|
+
data,
|
|
20827
|
+
...errors.length > 0 ? { errors } : {},
|
|
20828
|
+
status: discogsStatus(data, errors)
|
|
20829
|
+
};
|
|
20830
|
+
}
|
|
20831
|
+
function discogsStatus(data, errors) {
|
|
20832
|
+
if (Object.keys(data).length === 0) {
|
|
20833
|
+
return "error";
|
|
20834
|
+
}
|
|
20835
|
+
return errors.length > 0 ? "partial" : "ok";
|
|
20836
|
+
}
|
|
20837
|
+
function discogsSearchName(name) {
|
|
20838
|
+
return name.replace(/\s*\(\d+\)$/, "").replace(/^[^\p{L}\p{N}]+/u, "");
|
|
20839
|
+
}
|
|
20840
|
+
async function readDiscogsStyles(deps, releaseIds, labelName) {
|
|
20841
|
+
const headers = discogsHeaders(deps);
|
|
20842
|
+
const styled = new Map;
|
|
20843
|
+
const errors = [];
|
|
20844
|
+
let allCached = true;
|
|
20845
|
+
let viaSearch = 0;
|
|
20846
|
+
const searchUrl = `${DISCOGS_ROOT}/database/search?type=release&label=${encodeURIComponent(discogsSearchName(labelName))}&per_page=100`;
|
|
20847
|
+
try {
|
|
20848
|
+
const { cached, data } = await fetchEvidenceJson(deps.http, "discogs", searchUrl, headers);
|
|
20849
|
+
allCached &&= cached;
|
|
20850
|
+
for (const result of data.results ?? []) {
|
|
20851
|
+
if (typeof result.id === "number" && releaseIds.has(result.id) && !styled.has(result.id)) {
|
|
20852
|
+
styled.set(result.id, { genres: result.genre ?? [], styles: result.style ?? [] });
|
|
20853
|
+
}
|
|
20854
|
+
}
|
|
20855
|
+
viaSearch = styled.size;
|
|
20856
|
+
} catch (error) {
|
|
20857
|
+
errors.push(failureOf(error, searchUrl));
|
|
20858
|
+
}
|
|
20859
|
+
const target = Math.min(DISCOGS_STYLE_TARGET, releaseIds.size);
|
|
20860
|
+
const unread = [...releaseIds].filter((id) => !styled.has(id));
|
|
20861
|
+
let reads = 0;
|
|
20862
|
+
for (const id of unread) {
|
|
20863
|
+
if (styled.size >= target || reads >= DISCOGS_RELEASE_READS) {
|
|
20864
|
+
break;
|
|
20865
|
+
}
|
|
20866
|
+
reads += 1;
|
|
20867
|
+
const releaseUrl = `${DISCOGS_ROOT}/releases/${id}`;
|
|
20868
|
+
try {
|
|
20869
|
+
const { cached, data } = await fetchEvidenceJson(deps.http, "discogs", releaseUrl, headers);
|
|
20870
|
+
allCached &&= cached;
|
|
20871
|
+
styled.set(id, { genres: data.genres ?? [], styles: data.styles ?? [] });
|
|
20872
|
+
} catch (error) {
|
|
20873
|
+
errors.push(failureOf(error, releaseUrl));
|
|
20874
|
+
}
|
|
20875
|
+
}
|
|
20876
|
+
const rows = [...styled.values()];
|
|
20877
|
+
return {
|
|
20878
|
+
cached: allCached,
|
|
20879
|
+
errors,
|
|
20880
|
+
styles: {
|
|
20881
|
+
genres: tally(rows.flatMap((row) => row.genres)),
|
|
20882
|
+
labelReleasesListed: releaseIds.size,
|
|
20883
|
+
releasesRead: rows.length,
|
|
20884
|
+
styles: tally(rows.flatMap((row) => row.styles)).slice(0, 25),
|
|
20885
|
+
viaRelease: rows.length - viaSearch,
|
|
20886
|
+
viaSearch
|
|
20887
|
+
}
|
|
20888
|
+
};
|
|
20889
|
+
}
|
|
20890
|
+
function facetCounts(rows) {
|
|
20891
|
+
return (rows ?? []).flatMap((row) => row.name && typeof row.count === "number" ? [{ count: row.count, name: row.name }] : []).sort((a, b) => b.count - a.count || a.name.localeCompare(b.name));
|
|
20892
|
+
}
|
|
20893
|
+
function parseBeatportTracksPage(html) {
|
|
20894
|
+
const island = /<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/.exec(html);
|
|
20895
|
+
if (!island?.[1]) {
|
|
20896
|
+
return null;
|
|
20897
|
+
}
|
|
20898
|
+
let queries;
|
|
20899
|
+
try {
|
|
20900
|
+
const parsed = JSON.parse(island[1]);
|
|
20901
|
+
queries = parsed.props?.pageProps?.dehydratedState?.queries ?? [];
|
|
20902
|
+
} catch {
|
|
20903
|
+
return null;
|
|
20904
|
+
}
|
|
20905
|
+
const tracks = queries.find((query) => query.queryKey?.[0] === "tracks" && query.state?.data?.facets?.fields)?.state?.data;
|
|
20906
|
+
if (!tracks) {
|
|
20907
|
+
return null;
|
|
20908
|
+
}
|
|
20909
|
+
const fields = tracks.facets?.fields;
|
|
20910
|
+
const genres = facetCounts(fields?.genre);
|
|
20911
|
+
const total = genres.reduce((sum, genre) => sum + genre.count, 0);
|
|
20912
|
+
return {
|
|
20913
|
+
genres: genres.map((genre) => ({
|
|
20914
|
+
...genre,
|
|
20915
|
+
share: total > 0 ? Math.round(genre.count / total * 1000) / 1000 : 0
|
|
20916
|
+
})),
|
|
20917
|
+
subGenres: facetCounts(fields?.sub_genre),
|
|
20918
|
+
topArtists: facetCounts(fields?.artists).slice(0, 15),
|
|
20919
|
+
trackCount: tracks.count ?? total
|
|
20920
|
+
};
|
|
20921
|
+
}
|
|
20922
|
+
async function gatherBeatport(deps, mb) {
|
|
20923
|
+
if (!mb.label) {
|
|
20924
|
+
return {
|
|
20925
|
+
note: "MusicBrainz label unavailable, so no Beatport link to follow",
|
|
20926
|
+
status: "skipped"
|
|
20927
|
+
};
|
|
20928
|
+
}
|
|
20929
|
+
const labelUrl = beatportLabelUrlFrom(linkOf(mb.label.links, "beatport.com"));
|
|
20930
|
+
if (labelUrl === null) {
|
|
20931
|
+
return { note: "The MusicBrainz label has no Beatport link.", status: "no_link" };
|
|
20932
|
+
}
|
|
20933
|
+
if (deps.scraper === null) {
|
|
20934
|
+
return {
|
|
20935
|
+
note: "Beatport needs Firecrawl: set FIRECRAWL_API_KEY or install the firecrawl CLI.",
|
|
20936
|
+
status: "not_configured"
|
|
20937
|
+
};
|
|
20938
|
+
}
|
|
20939
|
+
const scraper = deps.scraper;
|
|
20940
|
+
const tracksUrl = `${labelUrl}/tracks`;
|
|
20941
|
+
try {
|
|
20942
|
+
const { cached, text } = await fetchEvidenceText(deps.http, "beatport", `SCRAPE ${tracksUrl}`, (signal) => scraper(tracksUrl, signal), (html) => parseBeatportTracksPage(html) !== null);
|
|
20943
|
+
const facts = parseBeatportTracksPage(text);
|
|
20944
|
+
if (facts === null) {
|
|
20945
|
+
return {
|
|
20946
|
+
errors: [
|
|
20947
|
+
{
|
|
20948
|
+
kind: "parse",
|
|
20949
|
+
message: "The Beatport page carried no tracks facet (a missing label or a layout change)",
|
|
20950
|
+
request: tracksUrl
|
|
20951
|
+
}
|
|
20952
|
+
],
|
|
20953
|
+
status: "error"
|
|
20954
|
+
};
|
|
20955
|
+
}
|
|
20956
|
+
return { cached, data: { ...facts, labelUrl, tracksUrl }, status: "ok" };
|
|
20957
|
+
} catch (error) {
|
|
20958
|
+
return { errors: [failureOf(error, tracksUrl)], status: "error" };
|
|
20959
|
+
}
|
|
20960
|
+
}
|
|
20961
|
+
async function gatherApple(deps, mb) {
|
|
20962
|
+
if (!mb.releases) {
|
|
20963
|
+
return {
|
|
20964
|
+
note: "MusicBrainz releases unavailable, so no barcodes to look up",
|
|
20965
|
+
status: "skipped"
|
|
20966
|
+
};
|
|
20967
|
+
}
|
|
20968
|
+
const seen = new Set;
|
|
20969
|
+
const targets = mb.releases.releases.filter((release) => {
|
|
20970
|
+
if (!release.barcode || seen.has(release.barcode)) {
|
|
20971
|
+
return false;
|
|
20972
|
+
}
|
|
20973
|
+
seen.add(release.barcode);
|
|
20974
|
+
return true;
|
|
20975
|
+
});
|
|
20976
|
+
if (targets.length === 0) {
|
|
20977
|
+
return { note: "No sampled MusicBrainz release carries a barcode.", status: "no_link" };
|
|
20978
|
+
}
|
|
20979
|
+
const releases = [];
|
|
20980
|
+
const errors = [];
|
|
20981
|
+
let allCached = true;
|
|
20982
|
+
for (const target of targets.slice(0, APPLE_LOOKUPS)) {
|
|
20983
|
+
const barcode = target.barcode ?? "";
|
|
20984
|
+
const url = `https://itunes.apple.com/lookup?upc=${encodeURIComponent(barcode)}&entity=album`;
|
|
20985
|
+
try {
|
|
20986
|
+
const { cached, data } = await fetchEvidenceJson(deps.http, "apple", url);
|
|
20987
|
+
allCached &&= cached;
|
|
20988
|
+
const hit = (data.results ?? []).find((result) => result.wrapperType === "collection");
|
|
20989
|
+
releases.push(hit ? {
|
|
20990
|
+
artistName: hit.artistName,
|
|
20991
|
+
barcode,
|
|
20992
|
+
collectionName: hit.collectionName,
|
|
20993
|
+
copyright: hit.copyright,
|
|
20994
|
+
found: true,
|
|
20995
|
+
mbTitle: target.title,
|
|
20996
|
+
primaryGenreName: hit.primaryGenreName
|
|
20997
|
+
} : { barcode, found: false, mbTitle: target.title });
|
|
20998
|
+
} catch (error) {
|
|
20999
|
+
errors.push(failureOf(error, url));
|
|
21000
|
+
}
|
|
21001
|
+
}
|
|
21002
|
+
const looked = Math.min(targets.length, APPLE_LOOKUPS);
|
|
21003
|
+
return {
|
|
21004
|
+
cached: allCached,
|
|
21005
|
+
data: {
|
|
21006
|
+
found: releases.filter((release) => release.found).length,
|
|
21007
|
+
genres: tally(releases.flatMap((release) => release.primaryGenreName ?? [])),
|
|
21008
|
+
looked,
|
|
21009
|
+
releases
|
|
21010
|
+
},
|
|
21011
|
+
...errors.length > 0 ? { errors } : {},
|
|
21012
|
+
status: statusFrom(looked, errors.length)
|
|
21013
|
+
};
|
|
21014
|
+
}
|
|
21015
|
+
function skipped() {
|
|
21016
|
+
return { note: "Not requested (--sources).", status: "skipped" };
|
|
21017
|
+
}
|
|
21018
|
+
async function resolveTarget(input, deps) {
|
|
21019
|
+
const trimmed = input.trim();
|
|
21020
|
+
if (MBID_PATTERN3.test(trimmed)) {
|
|
21021
|
+
return { input: trimmed, mbLabelId: trimmed.toLowerCase() };
|
|
21022
|
+
}
|
|
21023
|
+
const label = await deps.resolveLabel(trimmed);
|
|
21024
|
+
if (!label.mbLabelId) {
|
|
21025
|
+
throw new Error(`Label '${label.slug}' has no MusicBrainz identity (mb_label_id), so nothing says which same-named label to research. Resolve it by hand first (docs/label-entity.md).`);
|
|
21026
|
+
}
|
|
21027
|
+
return {
|
|
21028
|
+
id: label.id,
|
|
21029
|
+
input: trimmed,
|
|
21030
|
+
mbLabelId: label.mbLabelId.toLowerCase(),
|
|
21031
|
+
name: label.name,
|
|
21032
|
+
seedState: label.seedState,
|
|
21033
|
+
slug: label.slug
|
|
21034
|
+
};
|
|
21035
|
+
}
|
|
21036
|
+
async function gatherLabelEvidence(input, options, deps) {
|
|
21037
|
+
const label = await resolveTarget(input, deps);
|
|
21038
|
+
const wants = (source) => options.sources.includes(source);
|
|
21039
|
+
const needsMb = wants("musicbrainz") || wants("discogs") || wants("beatport") || wants("apple");
|
|
21040
|
+
const musicbrainz = needsMb ? await gatherMusicBrainz(deps, label.mbLabelId, options) : skipped();
|
|
21041
|
+
const mb = musicbrainz.data ?? {};
|
|
21042
|
+
const [discogs, beatport, apple] = await Promise.all([
|
|
21043
|
+
wants("discogs") ? gatherDiscogs(deps, mb) : Promise.resolve(skipped()),
|
|
21044
|
+
wants("beatport") ? gatherBeatport(deps, mb) : Promise.resolve(skipped()),
|
|
21045
|
+
wants("apple") ? gatherApple(deps, mb) : Promise.resolve(skipped())
|
|
21046
|
+
]);
|
|
21047
|
+
const sources = { apple, beatport, discogs, musicbrainz };
|
|
21048
|
+
const reports = Object.entries(sources);
|
|
21049
|
+
const errors = reports.flatMap(([source, report]) => (report.errors ?? []).map((failure) => ({ ...failure, source })));
|
|
21050
|
+
const requested = reports.filter(([source]) => wants(source));
|
|
21051
|
+
const answered = requested.filter(([, report]) => ["no_link", "ok", "partial"].includes(report.status));
|
|
21052
|
+
return {
|
|
21053
|
+
errors,
|
|
21054
|
+
fetchedAt: new Date(deps.http.now()).toISOString(),
|
|
21055
|
+
label: { ...label, name: label.name ?? mb.label?.name },
|
|
21056
|
+
ok: answered.length > 0,
|
|
21057
|
+
partial: answered.length < requested.length || errors.length > 0,
|
|
21058
|
+
requests: { cacheHits: deps.http.stats.cacheHits, network: deps.http.stats.requests },
|
|
21059
|
+
sources
|
|
21060
|
+
};
|
|
21061
|
+
}
|
|
21062
|
+
function firecrawlApiScraper(apiKey, http) {
|
|
21063
|
+
return async (url, signal) => {
|
|
21064
|
+
const response = await http.fetch(FIRECRAWL_SCRAPE_URL, {
|
|
21065
|
+
body: JSON.stringify({ formats: ["rawHtml"], url }),
|
|
21066
|
+
headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
|
|
21067
|
+
method: "POST",
|
|
21068
|
+
signal
|
|
21069
|
+
});
|
|
21070
|
+
if (!response.ok) {
|
|
21071
|
+
return { headers: response.headers, status: response.status, text: await response.text() };
|
|
21072
|
+
}
|
|
21073
|
+
const payload = await response.json();
|
|
21074
|
+
return {
|
|
21075
|
+
headers: response.headers,
|
|
21076
|
+
status: payload.data?.metadata?.statusCode ?? 200,
|
|
21077
|
+
text: payload.data?.rawHtml ?? ""
|
|
21078
|
+
};
|
|
21079
|
+
};
|
|
21080
|
+
}
|
|
21081
|
+
function firecrawlCliScraper(binary) {
|
|
21082
|
+
return (url, signal) => new Promise((resolve, reject) => {
|
|
21083
|
+
const child = spawn(binary, ["scrape", "-f", "rawHtml", url], {
|
|
21084
|
+
signal,
|
|
21085
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
21086
|
+
});
|
|
21087
|
+
const out = [];
|
|
21088
|
+
const err = [];
|
|
21089
|
+
child.stdout.on("data", (chunk) => out.push(chunk));
|
|
21090
|
+
child.stderr.on("data", (chunk) => err.push(chunk));
|
|
21091
|
+
child.on("error", reject);
|
|
21092
|
+
child.on("close", (code) => {
|
|
21093
|
+
if (code === 0) {
|
|
21094
|
+
resolve({
|
|
21095
|
+
headers: { get: () => null },
|
|
21096
|
+
status: 200,
|
|
21097
|
+
text: Buffer.concat(out).toString("utf8")
|
|
21098
|
+
});
|
|
21099
|
+
return;
|
|
21100
|
+
}
|
|
21101
|
+
const tail = Buffer.concat(err).toString("utf8").trim().split(`
|
|
21102
|
+
`).slice(-2).join(" ");
|
|
21103
|
+
reject(new Error(`firecrawl CLI exited ${code}: ${tail}`));
|
|
21104
|
+
});
|
|
21105
|
+
});
|
|
21106
|
+
}
|
|
21107
|
+
function defaultBeatportScraper(env, http) {
|
|
21108
|
+
if (env.firecrawlApiKey) {
|
|
21109
|
+
return firecrawlApiScraper(env.firecrawlApiKey, http);
|
|
21110
|
+
}
|
|
21111
|
+
return env.firecrawlBinary ? firecrawlCliScraper(env.firecrawlBinary) : null;
|
|
21112
|
+
}
|
|
21113
|
+
async function labelEvidenceCommand(input, options) {
|
|
21114
|
+
const [{ readOptionalEnv }, { resolveLabel }] = await Promise.all([
|
|
21115
|
+
Promise.resolve().then(() => (init_env(), exports_env)),
|
|
21116
|
+
Promise.resolve().then(() => (init_admin_labels3(), exports_admin_labels))
|
|
21117
|
+
]);
|
|
21118
|
+
const http = createEvidenceHttp({ refresh: options.refresh });
|
|
21119
|
+
const discogsToken = readOptionalEnv("DISCOGS_USER_TOKEN");
|
|
21120
|
+
if (discogsToken) {
|
|
21121
|
+
http.policies = { ...http.policies, discogs: { ...http.policies.discogs, intervalMs: 1100 } };
|
|
21122
|
+
}
|
|
21123
|
+
return gatherLabelEvidence(input, options, {
|
|
21124
|
+
discogsToken,
|
|
21125
|
+
http,
|
|
21126
|
+
resolveLabel,
|
|
21127
|
+
scraper: defaultBeatportScraper({
|
|
21128
|
+
firecrawlApiKey: readOptionalEnv("FIRECRAWL_API_KEY"),
|
|
21129
|
+
firecrawlBinary: Bun.which("firecrawl")
|
|
21130
|
+
}, http)
|
|
21131
|
+
});
|
|
21132
|
+
}
|
|
21133
|
+
function countLine(counts, limit) {
|
|
21134
|
+
return counts.slice(0, limit).map((entry) => `${entry.name} ${entry.count}`).join(", ");
|
|
21135
|
+
}
|
|
21136
|
+
function labelEvidenceLines(evidence) {
|
|
21137
|
+
const { apple, beatport, discogs, musicbrainz } = evidence.sources;
|
|
21138
|
+
const mb = musicbrainz.data;
|
|
21139
|
+
const name = evidence.label.name ?? evidence.label.mbLabelId;
|
|
21140
|
+
const lines = [`${name} (${evidence.label.mbLabelId})`];
|
|
21141
|
+
const mbParts = [];
|
|
21142
|
+
if (mb?.label) {
|
|
21143
|
+
mbParts.push([mb.label.type, mb.label.area, mb.label.lifeSpan.begin].filter(Boolean).join(", "));
|
|
21144
|
+
}
|
|
21145
|
+
if (mb?.releases) {
|
|
21146
|
+
const top = mb.releases.firstCreditArtists.slice(0, 6).map((artist) => `${artist.name} ${artist.releases}`).join(", ");
|
|
21147
|
+
mbParts.push(`${mb.releases.releaseCount} releases; first credits: ${top || "none"}`);
|
|
21148
|
+
}
|
|
21149
|
+
if (mb?.census) {
|
|
21150
|
+
mbParts.push(`census ${mb.census.recordingsCounted} recordings over ${mb.census.releasesRead} releases${mb.census.caveat ? ` (${mb.census.caveat})` : ""}`);
|
|
21151
|
+
}
|
|
21152
|
+
lines.push(` musicbrainz: ${musicbrainz.status}${mbParts.length ? ` — ${mbParts.join("; ")}` : ""}`);
|
|
21153
|
+
const dg = discogs.data;
|
|
21154
|
+
const dgDetail = dg?.styles ? `styles: ${countLine(dg.styles.styles, 6)} (${dg.styles.releasesRead} releases)` : dg?.candidates ? `unverified candidates: ${dg.candidates.map((c) => c.title).join(", ") || "none"}` : discogs.note;
|
|
21155
|
+
lines.push(` discogs: ${discogs.status}${dgDetail ? ` — ${dgDetail}` : ""}`);
|
|
21156
|
+
const bp = beatport.data;
|
|
21157
|
+
const bpDetail = bp ? `${bp.trackCount} tracks; genres: ${bp.genres.slice(0, 5).map((genre) => `${genre.name} ${Math.round(genre.share * 100)}%`).join(", ")}` : beatport.note;
|
|
21158
|
+
lines.push(` beatport: ${beatport.status}${bpDetail ? ` — ${bpDetail}` : ""}`);
|
|
21159
|
+
const ap = apple.data;
|
|
21160
|
+
const apDetail = ap ? `${ap.found}/${ap.looked} found; genres: ${countLine(ap.genres, 4) || "none"}` : apple.note;
|
|
21161
|
+
lines.push(` apple: ${apple.status}${apDetail ? ` — ${apDetail}` : ""}`);
|
|
21162
|
+
for (const failure of evidence.errors) {
|
|
21163
|
+
lines.push(` ! ${failure.source}: ${failure.message}`);
|
|
21164
|
+
}
|
|
21165
|
+
return lines;
|
|
21166
|
+
}
|
|
21167
|
+
var MBID_PATTERN3, MB_ROOT = "https://musicbrainz.org/ws/2", DISCOGS_ROOT = "https://api.discogs.com", FIRECRAWL_SCRAPE_URL = "https://api.firecrawl.dev/v2/scrape", VARIOUS_ARTISTS_MBID = "89ad4ac3-39f7-470e-963a-56509c546377", RELEASE_SAMPLE = 12, FIRST_CREDIT_ARTISTS = 25, APPLE_LOOKUPS = 5, DISCOGS_RELEASE_SAMPLE = 8, DISCOGS_STYLE_TARGET = 10, DISCOGS_RELEASE_READS = 8, PROFILE_CHARS = 800, EVIDENCE_SOURCES, MAX_CENSUS_PAGES = 5;
|
|
21168
|
+
var init_admin_label_evidence = __esm(() => {
|
|
21169
|
+
init_evidence_http();
|
|
21170
|
+
MBID_PATTERN3 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
21171
|
+
EVIDENCE_SOURCES = ["musicbrainz", "discogs", "beatport", "apple"];
|
|
21172
|
+
});
|
|
21173
|
+
|
|
19861
21174
|
// src/commands/admin-albums.ts
|
|
19862
21175
|
async function describeAlbumCommand(slug, options) {
|
|
19863
21176
|
return adminApiPost(`/api/v1/admin/albums/${encodeURIComponent(slug)}/bio`, buildBioBody2(options));
|
|
@@ -23703,6 +25016,25 @@ JSON field reference:
|
|
|
23703
25016
|
console.log(`${label.slug} ${label.seedState} ${looked ?? ""}`.trimEnd());
|
|
23704
25017
|
}
|
|
23705
25018
|
});
|
|
25019
|
+
labels.command("evidence").description("Gather a label's MusicBrainz, Discogs, Beatport and Apple evidence for triage (cached, rate-limited)").argument("<label>", "The label's MusicBrainz MBID, slug, or lbl_\u2026 id").option("--census", "Also count FIRST credits per recording over the label's releases", false).option("--census-pages <n>", "MusicBrainz pages the census may read, 1-5", "5").option("--sources <list>", "Comma list of musicbrainz,discogs,beatport,apple (default all)").option("--refresh", "Ignore cached responses and fetch fresh", false).option("--json", "Print JSON", false).action(async (input, options) => {
|
|
25020
|
+
await Promise.resolve().then(() => init_admin_label_evidence());
|
|
25021
|
+
const censusPages = parseCensusPages(options.censusPages);
|
|
25022
|
+
const evidence = await labelEvidenceCommand(input, {
|
|
25023
|
+
census: options.census,
|
|
25024
|
+
censusPages,
|
|
25025
|
+
refresh: options.refresh,
|
|
25026
|
+
sources: parseEvidenceSources(options.sources)
|
|
25027
|
+
});
|
|
25028
|
+
if (!evidence.ok) {
|
|
25029
|
+
process.exitCode = 1;
|
|
25030
|
+
}
|
|
25031
|
+
if (options.json) {
|
|
25032
|
+
printJson(evidence);
|
|
25033
|
+
return;
|
|
25034
|
+
}
|
|
25035
|
+
console.log(labelEvidenceLines(evidence).join(`
|
|
25036
|
+
`));
|
|
25037
|
+
});
|
|
23706
25038
|
labels.command("triage").description("Record what a triage round found for a label (agent; never rules)").argument("<slug>", "The label slug").requiredOption("--payload <file>", "JSON file holding the round's finding for this label").option("--json", "Print JSON", false).action(async (slug, options) => {
|
|
23707
25039
|
const payload = JSON.parse(readFileSync6(options.payload, "utf8"));
|
|
23708
25040
|
await Promise.resolve().then(() => init_admin_labels2());
|
|
@@ -26691,6 +28023,7 @@ var stringOptions = new Set([
|
|
|
26691
28023
|
"--bpm-confidence",
|
|
26692
28024
|
"--bpm-source",
|
|
26693
28025
|
"--calls",
|
|
28026
|
+
"--census-pages",
|
|
26694
28027
|
"--composition",
|
|
26695
28028
|
"--consumer-digest",
|
|
26696
28029
|
"--consumer-item-count",
|
|
@@ -26769,6 +28102,7 @@ var stringOptions = new Set([
|
|
|
26769
28102
|
"--since",
|
|
26770
28103
|
"--soundcloud-url",
|
|
26771
28104
|
"--source",
|
|
28105
|
+
"--sources",
|
|
26772
28106
|
"--status",
|
|
26773
28107
|
"--stale-before",
|
|
26774
28108
|
"--stream",
|