@enricai/barnacle 1.6.9 → 1.6.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -3
- package/dist/api/routes/submissions.d.ts +21 -0
- package/dist/api/routes/submissions.d.ts.map +1 -0
- package/dist/api/routes/submissions.js +48 -0
- package/dist/api/routes/submissions.js.map +1 -0
- package/dist/api/schemas/submissions.d.ts +110 -0
- package/dist/api/schemas/submissions.d.ts.map +1 -0
- package/dist/api/schemas/submissions.js +77 -0
- package/dist/api/schemas/submissions.js.map +1 -0
- package/dist/config.d.ts +34 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +5 -0
- package/dist/config.js.map +1 -1
- package/dist/lib/telemetry/beacon-capture.d.ts +37 -0
- package/dist/lib/telemetry/beacon-capture.d.ts.map +1 -0
- package/dist/lib/telemetry/beacon-capture.js +47 -0
- package/dist/lib/telemetry/beacon-capture.js.map +1 -0
- package/dist/lib/telemetry/reconciliation-record.d.ts +105 -0
- package/dist/lib/telemetry/reconciliation-record.d.ts.map +1 -0
- package/dist/lib/telemetry/reconciliation-record.js +76 -0
- package/dist/lib/telemetry/reconciliation-record.js.map +1 -0
- package/dist/lib/telemetry/reconciliation-source.d.ts +33 -0
- package/dist/lib/telemetry/reconciliation-source.d.ts.map +1 -0
- package/dist/lib/telemetry/reconciliation-source.js +91 -0
- package/dist/lib/telemetry/reconciliation-source.js.map +1 -0
- package/dist/lib/telemetry/s3-sink.d.ts.map +1 -1
- package/dist/lib/telemetry/s3-sink.js +1 -2
- package/dist/lib/telemetry/s3-sink.js.map +1 -1
- package/dist/lib/telemetry/submission-capture.d.ts +30 -9
- package/dist/lib/telemetry/submission-capture.d.ts.map +1 -1
- package/dist/lib/telemetry/submission-capture.js +32 -21
- package/dist/lib/telemetry/submission-capture.js.map +1 -1
- package/dist/lib/telemetry/submission-query.d.ts +32 -0
- package/dist/lib/telemetry/submission-query.d.ts.map +1 -0
- package/dist/lib/telemetry/submission-query.js +49 -0
- package/dist/lib/telemetry/submission-query.js.map +1 -0
- package/dist/lib/telemetry/submission-reader.d.ts +52 -0
- package/dist/lib/telemetry/submission-reader.d.ts.map +1 -0
- package/dist/lib/telemetry/submission-reader.js +111 -0
- package/dist/lib/telemetry/submission-reader.js.map +1 -0
- package/dist/lib/telemetry/submissions-s3-objects.d.ts +23 -0
- package/dist/lib/telemetry/submissions-s3-objects.d.ts.map +1 -0
- package/dist/lib/telemetry/submissions-s3-objects.js +132 -0
- package/dist/lib/telemetry/submissions-s3-objects.js.map +1 -0
- package/dist/lib/telemetry/submissions-s3-reader.d.ts +25 -0
- package/dist/lib/telemetry/submissions-s3-reader.d.ts.map +1 -0
- package/dist/lib/telemetry/submissions-s3-reader.js +63 -0
- package/dist/lib/telemetry/submissions-s3-reader.js.map +1 -0
- package/dist/lib/tracking-click.d.ts +13 -2
- package/dist/lib/tracking-click.d.ts.map +1 -1
- package/dist/lib/tracking-click.js +42 -4
- package/dist/lib/tracking-click.js.map +1 -1
- package/dist/plugins/loader.d.ts +17 -3
- package/dist/plugins/loader.d.ts.map +1 -1
- package/dist/plugins/loader.js +59 -6
- package/dist/plugins/loader.js.map +1 -1
- package/dist/scraper/cookie-jar.d.ts +10 -4
- package/dist/scraper/cookie-jar.d.ts.map +1 -1
- package/dist/scraper/cookie-jar.js +10 -5
- package/dist/scraper/cookie-jar.js.map +1 -1
- package/dist/scraper/deep-locator-candidates.d.ts +84 -0
- package/dist/scraper/deep-locator-candidates.d.ts.map +1 -0
- package/dist/scraper/deep-locator-candidates.js +206 -0
- package/dist/scraper/deep-locator-candidates.js.map +1 -0
- package/dist/scraper/deep-locator-fake.d.ts +116 -0
- package/dist/scraper/deep-locator-fake.d.ts.map +1 -0
- package/dist/scraper/deep-locator-fake.js +128 -0
- package/dist/scraper/deep-locator-fake.js.map +1 -0
- package/dist/scraper/flow-runner.d.ts +28 -1
- package/dist/scraper/flow-runner.d.ts.map +1 -1
- package/dist/scraper/flow-runner.js +204 -14
- package/dist/scraper/flow-runner.js.map +1 -1
- package/dist/scraper/frame-target.d.ts +60 -14
- package/dist/scraper/frame-target.d.ts.map +1 -1
- package/dist/scraper/frame-target.js +126 -38
- package/dist/scraper/frame-target.js.map +1 -1
- package/dist/scraper/stagehand-guard.d.ts +14 -0
- package/dist/scraper/stagehand-guard.d.ts.map +1 -1
- package/dist/scraper/stagehand-guard.js +28 -8
- package/dist/scraper/stagehand-guard.js.map +1 -1
- package/dist/scraper/watchdog.d.ts +38 -0
- package/dist/scraper/watchdog.d.ts.map +1 -0
- package/dist/scraper/watchdog.js +54 -0
- package/dist/scraper/watchdog.js.map +1 -0
- package/dist/scripts/recon-browser.d.ts.map +1 -1
- package/dist/scripts/recon-browser.js +14 -3
- package/dist/scripts/recon-browser.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +2 -0
- package/dist/server.js.map +1 -1
- package/dist/site-plugin.d.ts +12 -0
- package/dist/site-plugin.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Schemas for the durable per-run reconciliation record. Extends the submit
|
|
3
|
+
* envelope with an opaque `joinKeys` bag and adds a distinct
|
|
4
|
+
* beacon/conversion-event kind, so a plugin can reconcile its own attribution
|
|
5
|
+
* provider's payment per-run instead of only at the cohort level. `joinKeys`
|
|
6
|
+
* is deliberately untyped `Record<string, unknown>` (matching
|
|
7
|
+
* `SitePluginResult.auditPayload`'s "opaque, plugin-owned shape" precedent in
|
|
8
|
+
* `src/site-plugin.ts`) — core has no business knowing what a given
|
|
9
|
+
* attribution vendor calls its join key.
|
|
10
|
+
*
|
|
11
|
+
* Both `submission-capture.ts` and `s3-sink.ts` are strictly append-only, so
|
|
12
|
+
* the beacon outcome — which resolves after dispatch already returned and
|
|
13
|
+
* wrote its submit line — is recorded as its own later line rather than a
|
|
14
|
+
* mutation of the submit line. A reader folds the two kinds together by
|
|
15
|
+
* `requestId` to reconstruct one reconciliation record per run.
|
|
16
|
+
*
|
|
17
|
+
* `kind` defaults to `"submit"` because `.barnacle/submissions.ndjson`
|
|
18
|
+
* already holds historical unkinded lines that `s3-sink.ts` has shipped to
|
|
19
|
+
* S3 — a required `kind` would make every one of those lines unparseable.
|
|
20
|
+
*/
|
|
21
|
+
import { z } from "zod/v4";
|
|
22
|
+
/**
|
|
23
|
+
* Submit-side record: the existing submission envelope shape plus an opaque
|
|
24
|
+
* `joinKeys` bag. Nullable-with-default so pre-existing lines that never
|
|
25
|
+
* carried this field still parse.
|
|
26
|
+
*/
|
|
27
|
+
export declare const submitRecordSchema: z.ZodObject<{
|
|
28
|
+
kind: z.ZodDefault<z.ZodLiteral<"submit">>;
|
|
29
|
+
siteId: z.ZodString;
|
|
30
|
+
requestId: z.ZodString;
|
|
31
|
+
joinKeys: z.ZodDefault<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
|
|
32
|
+
inboundPayload: z.ZodUnknown;
|
|
33
|
+
status: z.ZodEnum<{
|
|
34
|
+
error: "error";
|
|
35
|
+
submitted: "submitted";
|
|
36
|
+
}>;
|
|
37
|
+
auditPayload: z.ZodUnknown;
|
|
38
|
+
errorMessage: z.ZodNullable<z.ZodString>;
|
|
39
|
+
durationMs: z.ZodNumber;
|
|
40
|
+
ts: z.ZodString;
|
|
41
|
+
}, z.core.$strip>;
|
|
42
|
+
export type SubmitRecord = z.infer<typeof submitRecordSchema>;
|
|
43
|
+
/**
|
|
44
|
+
* Beacon/conversion-event record: fired independently of, and later than,
|
|
45
|
+
* the submit record for the same `requestId`. Models "submitted but the
|
|
46
|
+
* beacon did not fire" as a distinct dimension from submit outcome.
|
|
47
|
+
* `beaconStatus: "skipped"` means no beacon was ever applicable for this run
|
|
48
|
+
* (no usable TrackingUrl, or the plugin manages its own beacon nav — see
|
|
49
|
+
* `dispatch()` in `src/plugins/loader.ts`) — distinct from `"not_fired"`
|
|
50
|
+
* (the reader's fold default for a submit row with no matching beacon line
|
|
51
|
+
* at all), so "submitted but the beacon did not fire" excludes runs that
|
|
52
|
+
* never had a beacon to fire.
|
|
53
|
+
*/
|
|
54
|
+
export declare const beaconEventSchema: z.ZodObject<{
|
|
55
|
+
kind: z.ZodLiteral<"beacon">;
|
|
56
|
+
requestId: z.ZodString;
|
|
57
|
+
siteId: z.ZodString;
|
|
58
|
+
joinKeys: z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
59
|
+
beaconStatus: z.ZodEnum<{
|
|
60
|
+
failed: "failed";
|
|
61
|
+
fired: "fired";
|
|
62
|
+
skipped: "skipped";
|
|
63
|
+
}>;
|
|
64
|
+
trackingUrl: z.ZodNullable<z.ZodString>;
|
|
65
|
+
durationMs: z.ZodNumber;
|
|
66
|
+
ts: z.ZodString;
|
|
67
|
+
}, z.core.$strip>;
|
|
68
|
+
export type BeaconEvent = z.infer<typeof beaconEventSchema>;
|
|
69
|
+
/**
|
|
70
|
+
* Routes an NDJSON line to `submitRecordSchema` or `beaconEventSchema` by
|
|
71
|
+
* `kind`. Injects `kind: "submit"` ahead of the discriminated union when the
|
|
72
|
+
* field is absent, because `z.discriminatedUnion` reads the raw input's
|
|
73
|
+
* discriminant directly and won't fall back to a member schema's default —
|
|
74
|
+
* confirmed against zod/v4 3.25.76.
|
|
75
|
+
*/
|
|
76
|
+
export declare const reconciliationRecordSchema: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
77
|
+
kind: z.ZodDefault<z.ZodLiteral<"submit">>;
|
|
78
|
+
siteId: z.ZodString;
|
|
79
|
+
requestId: z.ZodString;
|
|
80
|
+
joinKeys: z.ZodDefault<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
|
|
81
|
+
inboundPayload: z.ZodUnknown;
|
|
82
|
+
status: z.ZodEnum<{
|
|
83
|
+
error: "error";
|
|
84
|
+
submitted: "submitted";
|
|
85
|
+
}>;
|
|
86
|
+
auditPayload: z.ZodUnknown;
|
|
87
|
+
errorMessage: z.ZodNullable<z.ZodString>;
|
|
88
|
+
durationMs: z.ZodNumber;
|
|
89
|
+
ts: z.ZodString;
|
|
90
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
91
|
+
kind: z.ZodLiteral<"beacon">;
|
|
92
|
+
requestId: z.ZodString;
|
|
93
|
+
siteId: z.ZodString;
|
|
94
|
+
joinKeys: z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
95
|
+
beaconStatus: z.ZodEnum<{
|
|
96
|
+
failed: "failed";
|
|
97
|
+
fired: "fired";
|
|
98
|
+
skipped: "skipped";
|
|
99
|
+
}>;
|
|
100
|
+
trackingUrl: z.ZodNullable<z.ZodString>;
|
|
101
|
+
durationMs: z.ZodNumber;
|
|
102
|
+
ts: z.ZodString;
|
|
103
|
+
}, z.core.$strip>]>>;
|
|
104
|
+
export type ReconciliationRecord = z.infer<typeof reconciliationRecordSchema>;
|
|
105
|
+
//# sourceMappingURL=reconciliation-record.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reconciliation-record.d.ts","sourceRoot":"","sources":["../../../src/lib/telemetry/reconciliation-record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAE3B;;;;GAIG;AACH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;iBAW7B,CAAC;AAEH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D;;;;;;;;;;GAUG;AACH,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;iBAS5B,CAAC;AAEH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D;;;;;;GAMG;AACH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;oBAQtC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Schemas for the durable per-run reconciliation record. Extends the submit
|
|
4
|
+
* envelope with an opaque `joinKeys` bag and adds a distinct
|
|
5
|
+
* beacon/conversion-event kind, so a plugin can reconcile its own attribution
|
|
6
|
+
* provider's payment per-run instead of only at the cohort level. `joinKeys`
|
|
7
|
+
* is deliberately untyped `Record<string, unknown>` (matching
|
|
8
|
+
* `SitePluginResult.auditPayload`'s "opaque, plugin-owned shape" precedent in
|
|
9
|
+
* `src/site-plugin.ts`) — core has no business knowing what a given
|
|
10
|
+
* attribution vendor calls its join key.
|
|
11
|
+
*
|
|
12
|
+
* Both `submission-capture.ts` and `s3-sink.ts` are strictly append-only, so
|
|
13
|
+
* the beacon outcome — which resolves after dispatch already returned and
|
|
14
|
+
* wrote its submit line — is recorded as its own later line rather than a
|
|
15
|
+
* mutation of the submit line. A reader folds the two kinds together by
|
|
16
|
+
* `requestId` to reconstruct one reconciliation record per run.
|
|
17
|
+
*
|
|
18
|
+
* `kind` defaults to `"submit"` because `.barnacle/submissions.ndjson`
|
|
19
|
+
* already holds historical unkinded lines that `s3-sink.ts` has shipped to
|
|
20
|
+
* S3 — a required `kind` would make every one of those lines unparseable.
|
|
21
|
+
*/
|
|
22
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
|
+
exports.reconciliationRecordSchema = exports.beaconEventSchema = exports.submitRecordSchema = void 0;
|
|
24
|
+
const v4_1 = require("zod/v4");
|
|
25
|
+
/**
|
|
26
|
+
* Submit-side record: the existing submission envelope shape plus an opaque
|
|
27
|
+
* `joinKeys` bag. Nullable-with-default so pre-existing lines that never
|
|
28
|
+
* carried this field still parse.
|
|
29
|
+
*/
|
|
30
|
+
exports.submitRecordSchema = v4_1.z.object({
|
|
31
|
+
kind: v4_1.z.literal("submit").default("submit"),
|
|
32
|
+
siteId: v4_1.z.string(),
|
|
33
|
+
requestId: v4_1.z.string(),
|
|
34
|
+
joinKeys: v4_1.z.record(v4_1.z.string(), v4_1.z.unknown()).nullable().default(null),
|
|
35
|
+
inboundPayload: v4_1.z.unknown(),
|
|
36
|
+
status: v4_1.z.enum(["submitted", "error"]),
|
|
37
|
+
auditPayload: v4_1.z.unknown(),
|
|
38
|
+
errorMessage: v4_1.z.string().nullable(),
|
|
39
|
+
durationMs: v4_1.z.number(),
|
|
40
|
+
ts: v4_1.z.string(),
|
|
41
|
+
});
|
|
42
|
+
/**
|
|
43
|
+
* Beacon/conversion-event record: fired independently of, and later than,
|
|
44
|
+
* the submit record for the same `requestId`. Models "submitted but the
|
|
45
|
+
* beacon did not fire" as a distinct dimension from submit outcome.
|
|
46
|
+
* `beaconStatus: "skipped"` means no beacon was ever applicable for this run
|
|
47
|
+
* (no usable TrackingUrl, or the plugin manages its own beacon nav — see
|
|
48
|
+
* `dispatch()` in `src/plugins/loader.ts`) — distinct from `"not_fired"`
|
|
49
|
+
* (the reader's fold default for a submit row with no matching beacon line
|
|
50
|
+
* at all), so "submitted but the beacon did not fire" excludes runs that
|
|
51
|
+
* never had a beacon to fire.
|
|
52
|
+
*/
|
|
53
|
+
exports.beaconEventSchema = v4_1.z.object({
|
|
54
|
+
kind: v4_1.z.literal("beacon"),
|
|
55
|
+
requestId: v4_1.z.string(),
|
|
56
|
+
siteId: v4_1.z.string(),
|
|
57
|
+
joinKeys: v4_1.z.record(v4_1.z.string(), v4_1.z.unknown()).nullable(),
|
|
58
|
+
beaconStatus: v4_1.z.enum(["fired", "failed", "skipped"]),
|
|
59
|
+
trackingUrl: v4_1.z.string().nullable(),
|
|
60
|
+
durationMs: v4_1.z.number(),
|
|
61
|
+
ts: v4_1.z.string(),
|
|
62
|
+
});
|
|
63
|
+
/**
|
|
64
|
+
* Routes an NDJSON line to `submitRecordSchema` or `beaconEventSchema` by
|
|
65
|
+
* `kind`. Injects `kind: "submit"` ahead of the discriminated union when the
|
|
66
|
+
* field is absent, because `z.discriminatedUnion` reads the raw input's
|
|
67
|
+
* discriminant directly and won't fall back to a member schema's default —
|
|
68
|
+
* confirmed against zod/v4 3.25.76.
|
|
69
|
+
*/
|
|
70
|
+
exports.reconciliationRecordSchema = v4_1.z.preprocess((value) => {
|
|
71
|
+
if (typeof value === "object" && value !== null && !("kind" in value)) {
|
|
72
|
+
return { ...value, kind: "submit" };
|
|
73
|
+
}
|
|
74
|
+
return value;
|
|
75
|
+
}, v4_1.z.discriminatedUnion("kind", [exports.submitRecordSchema, exports.beaconEventSchema]));
|
|
76
|
+
//# sourceMappingURL=reconciliation-record.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reconciliation-record.js","sourceRoot":"","sources":["../../../src/lib/telemetry/reconciliation-record.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;GAmBG;;;AAEH,+BAA2B;AAE3B;;;;GAIG;AACU,QAAA,kBAAkB,GAAG,MAAC,CAAC,MAAM,CAAC;IACzC,IAAI,EAAE,MAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;IAC3C,MAAM,EAAE,MAAC,CAAC,MAAM,EAAE;IAClB,SAAS,EAAE,MAAC,CAAC,MAAM,EAAE;IACrB,QAAQ,EAAE,MAAC,CAAC,MAAM,CAAC,MAAC,CAAC,MAAM,EAAE,EAAE,MAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACpE,cAAc,EAAE,MAAC,CAAC,OAAO,EAAE;IAC3B,MAAM,EAAE,MAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACtC,YAAY,EAAE,MAAC,CAAC,OAAO,EAAE;IACzB,YAAY,EAAE,MAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACnC,UAAU,EAAE,MAAC,CAAC,MAAM,EAAE;IACtB,EAAE,EAAE,MAAC,CAAC,MAAM,EAAE;CACf,CAAC,CAAC;AAIH;;;;;;;;;;GAUG;AACU,QAAA,iBAAiB,GAAG,MAAC,CAAC,MAAM,CAAC;IACxC,IAAI,EAAE,MAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;IACzB,SAAS,EAAE,MAAC,CAAC,MAAM,EAAE;IACrB,MAAM,EAAE,MAAC,CAAC,MAAM,EAAE;IAClB,QAAQ,EAAE,MAAC,CAAC,MAAM,CAAC,MAAC,CAAC,MAAM,EAAE,EAAE,MAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACtD,YAAY,EAAE,MAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IACpD,WAAW,EAAE,MAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,UAAU,EAAE,MAAC,CAAC,MAAM,EAAE;IACtB,EAAE,EAAE,MAAC,CAAC,MAAM,EAAE;CACf,CAAC,CAAC;AAIH;;;;;;GAMG;AACU,QAAA,0BAA0B,GAAG,MAAC,CAAC,UAAU,CACpD,CAAC,KAAK,EAAE,EAAE;IACR,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,EAAE,CAAC;QACtE,OAAO,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IACtC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC,EACD,MAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,QAAA,kBAAkB,EAAE,QAAA,iBAAiB,CAAC,CAAC,CACtE,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable read path for reconciliation rows: unions the container-local
|
|
3
|
+
* submissions sink with the S3-mirrored objects (feat-003's
|
|
4
|
+
* `listSubmissionsS3Objects` + `fetchSubmissionsS3Records`) before folding,
|
|
5
|
+
* so a submit line written by one ECS task and its beacon line written by
|
|
6
|
+
* another both land in the same row instead of two ephemeral, incomplete
|
|
7
|
+
* ones. Merge happens at the record level — before
|
|
8
|
+
* `foldReconciliationRecords` — because folding first would discard the
|
|
9
|
+
* beacon detail a cross-store fold still needs (see
|
|
10
|
+
* `submission-reader.ts`'s `foldReconciliationRecords`).
|
|
11
|
+
*
|
|
12
|
+
* The local sink and its S3 mirror both receive the exact same line for
|
|
13
|
+
* every event (`appendSubmissionSinkLine` writes to disk and buffers to S3
|
|
14
|
+
* in the same call), so an overlap between the two stores is an exact JSON
|
|
15
|
+
* duplicate, never a conflicting value — deduping on `kind:requestId:ts` is
|
|
16
|
+
* therefore lossless.
|
|
17
|
+
*/
|
|
18
|
+
import { type ReconciliationRow } from "../../lib/telemetry/submission-reader";
|
|
19
|
+
import { type ListSubmissionsS3ObjectsOptions } from "../../lib/telemetry/submissions-s3-objects";
|
|
20
|
+
/** Options for `readDurableReconciliationRows`. */
|
|
21
|
+
export interface ReadDurableReconciliationRowsOptions extends ListSubmissionsS3ObjectsOptions {
|
|
22
|
+
/** Override the local sink path; used in tests to avoid touching the real file. */
|
|
23
|
+
sinkPath?: string;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Reads and folds the complete cross-container reconciliation history: the
|
|
27
|
+
* local sink plus its S3 mirror, unioned and deduped before folding, so
|
|
28
|
+
* `GET /v1/submissions` answers from durable storage instead of one ECS
|
|
29
|
+
* task's ephemeral file. Falls back to local-only rows when no bucket is
|
|
30
|
+
* configured or the S3 read fails.
|
|
31
|
+
*/
|
|
32
|
+
export declare function readDurableReconciliationRows(opts?: ReadDurableReconciliationRowsOptions): Promise<ReconciliationRow[]>;
|
|
33
|
+
//# sourceMappingURL=reconciliation-source.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reconciliation-source.d.ts","sourceRoot":"","sources":["../../../src/lib/telemetry/reconciliation-source.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AASH,OAAO,EAGL,KAAK,iBAAiB,EACvB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EACL,KAAK,+BAA+B,EAErC,MAAM,wCAAwC,CAAC;AAKhD,mDAAmD;AACnD,MAAM,WAAW,oCAAqC,SAAQ,+BAA+B;IAC3F,mFAAmF;IACnF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAwDD;;;;;;GAMG;AACH,wBAAsB,6BAA6B,CACjD,IAAI,GAAE,oCAAyC,GAC9C,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAS9B"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Durable read path for reconciliation rows: unions the container-local
|
|
4
|
+
* submissions sink with the S3-mirrored objects (feat-003's
|
|
5
|
+
* `listSubmissionsS3Objects` + `fetchSubmissionsS3Records`) before folding,
|
|
6
|
+
* so a submit line written by one ECS task and its beacon line written by
|
|
7
|
+
* another both land in the same row instead of two ephemeral, incomplete
|
|
8
|
+
* ones. Merge happens at the record level — before
|
|
9
|
+
* `foldReconciliationRecords` — because folding first would discard the
|
|
10
|
+
* beacon detail a cross-store fold still needs (see
|
|
11
|
+
* `submission-reader.ts`'s `foldReconciliationRecords`).
|
|
12
|
+
*
|
|
13
|
+
* The local sink and its S3 mirror both receive the exact same line for
|
|
14
|
+
* every event (`appendSubmissionSinkLine` writes to disk and buffers to S3
|
|
15
|
+
* in the same call), so an overlap between the two stores is an exact JSON
|
|
16
|
+
* duplicate, never a conflicting value — deduping on `kind:requestId:ts` is
|
|
17
|
+
* therefore lossless.
|
|
18
|
+
*/
|
|
19
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
20
|
+
exports.readDurableReconciliationRows = readDurableReconciliationRows;
|
|
21
|
+
const node_fs_1 = require("node:fs");
|
|
22
|
+
const promises_1 = require("node:fs/promises");
|
|
23
|
+
const config_1 = require("../../config");
|
|
24
|
+
const errors_1 = require("../../lib/errors");
|
|
25
|
+
const logging_1 = require("../../lib/logging");
|
|
26
|
+
const submission_reader_1 = require("../../lib/telemetry/submission-reader");
|
|
27
|
+
const submissions_s3_objects_1 = require("../../lib/telemetry/submissions-s3-objects");
|
|
28
|
+
const submissions_s3_reader_1 = require("../../lib/telemetry/submissions-s3-reader");
|
|
29
|
+
const logger = (0, logging_1.getLogger)({ name: "telemetry/reconciliation-source" });
|
|
30
|
+
/**
|
|
31
|
+
* Reads the local sink into raw records, exactly as `readReconciliationRows`
|
|
32
|
+
* does (`existsSync` + `readFile` + `parseReconciliationLines`), rather than
|
|
33
|
+
* calling `readReconciliationRows` itself — that helper returns already-folded
|
|
34
|
+
* rows, which would drop the beacon detail a cross-store fold needs.
|
|
35
|
+
*/
|
|
36
|
+
async function readLocalRecords(sinkPath) {
|
|
37
|
+
if (!(0, node_fs_1.existsSync)(sinkPath))
|
|
38
|
+
return [];
|
|
39
|
+
const content = await (0, promises_1.readFile)(sinkPath, "utf8");
|
|
40
|
+
return (0, submission_reader_1.parseReconciliationLines)(content);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Lists and fetches the S3-mirrored records for the window. A rejection
|
|
44
|
+
* anywhere in the S3 path (listing or fetch) is logged and swallowed rather
|
|
45
|
+
* than propagated, so an S3 outage degrades the answer to local-only instead
|
|
46
|
+
* of failing the whole read.
|
|
47
|
+
*/
|
|
48
|
+
async function readS3Records(opts) {
|
|
49
|
+
try {
|
|
50
|
+
const keys = await (0, submissions_s3_objects_1.listSubmissionsS3Objects)(opts);
|
|
51
|
+
if (keys.length === 0)
|
|
52
|
+
return [];
|
|
53
|
+
return await (0, submissions_s3_reader_1.fetchSubmissionsS3Records)(keys);
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
logger.warn(`reconciliation-source: S3 read failed, falling back to local-only rows: ${(0, errors_1.toErrorMessage)(err)}`);
|
|
57
|
+
return [];
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/** Dedupe key for a raw record: the same line written to both stores collapses to one. */
|
|
61
|
+
function dedupeKey(record) {
|
|
62
|
+
return `${record.kind}:${record.requestId}:${record.ts}`;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Unions the two record sets, local first, and dedupes on
|
|
66
|
+
* `kind:requestId:ts` — the same line written to both stores collapses to
|
|
67
|
+
* one entry, so `foldReconciliationRecords` sees it exactly once.
|
|
68
|
+
*/
|
|
69
|
+
function mergeRecords(localRecords, s3Records) {
|
|
70
|
+
const deduped = new Map();
|
|
71
|
+
for (const record of [...localRecords, ...s3Records]) {
|
|
72
|
+
deduped.set(dedupeKey(record), record);
|
|
73
|
+
}
|
|
74
|
+
return Array.from(deduped.values());
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Reads and folds the complete cross-container reconciliation history: the
|
|
78
|
+
* local sink plus its S3 mirror, unioned and deduped before folding, so
|
|
79
|
+
* `GET /v1/submissions` answers from durable storage instead of one ECS
|
|
80
|
+
* task's ephemeral file. Falls back to local-only rows when no bucket is
|
|
81
|
+
* configured or the S3 read fails.
|
|
82
|
+
*/
|
|
83
|
+
async function readDurableReconciliationRows(opts = {}) {
|
|
84
|
+
const { sinkPath = config_1.config.telemetry.submissionsNdjsonPath, ...windowOpts } = opts;
|
|
85
|
+
const [localRecords, s3Records] = await Promise.all([
|
|
86
|
+
readLocalRecords(sinkPath),
|
|
87
|
+
readS3Records(windowOpts),
|
|
88
|
+
]);
|
|
89
|
+
return (0, submission_reader_1.foldReconciliationRecords)(mergeRecords(localRecords, s3Records));
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=reconciliation-source.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reconciliation-source.js","sourceRoot":"","sources":["../../../src/lib/telemetry/reconciliation-source.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;GAgBG;;;AAEH,qCAAqC;AACrC,+CAA4C;AAE5C,qCAAkC;AAClC,yCAA8C;AAC9C,2CAA0C;AAE1C,yEAI2C;AAC3C,mFAGgD;AAChD,iFAAkF;AAElF,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,iCAAiC,EAAE,CAAC,CAAC;AAQtE;;;;;GAKG;AACH,KAAK,UAAU,gBAAgB,CAAC,QAAgB;IAC9C,IAAI,CAAC,IAAA,oBAAU,EAAC,QAAQ,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,MAAM,OAAO,GAAG,MAAM,IAAA,mBAAQ,EAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACjD,OAAO,IAAA,4CAAwB,EAAC,OAAO,CAAC,CAAC;AAC3C,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,aAAa,CAC1B,IAAqC;IAErC,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,IAAA,iDAAwB,EAAC,IAAI,CAAC,CAAC;QAClD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACjC,OAAO,MAAM,IAAA,iDAAyB,EAAC,IAAI,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CACT,2EAA2E,IAAA,uBAAc,EAAC,GAAG,CAAC,EAAE,CACjG,CAAC;QACF,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,0FAA0F;AAC1F,SAAS,SAAS,CAAC,MAA4B;IAC7C,OAAO,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;AAC3D,CAAC;AAED;;;;GAIG;AACH,SAAS,YAAY,CACnB,YAAoC,EACpC,SAAiC;IAEjC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAgC,CAAC;IACxD,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,YAAY,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;AACtC,CAAC;AAED;;;;;;GAMG;AACI,KAAK,wCACV,IAAI,GAAyC,EAAE;IAE/C,MAAM,EAAE,QAAQ,GAAG,eAAM,CAAC,SAAS,CAAC,qBAAqB,EAAE,GAAG,UAAU,EAAE,GAAG,IAAI,CAAC;IAElF,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAClD,gBAAgB,CAAC,QAAQ,CAAC;QAC1B,aAAa,CAAC,UAAU,CAAC;KAC1B,CAAC,CAAC;IAEH,OAAO,IAAA,6CAAyB,EAAC,YAAY,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC,CAAC;AAC1E,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3-sink.d.ts","sourceRoot":"","sources":["../../../src/lib/telemetry/s3-sink.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;
|
|
1
|
+
{"version":3,"file":"s3-sink.d.ts","sourceRoot":"","sources":["../../../src/lib/telemetry/s3-sink.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAsDH;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAIjD;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAIvD;AA0CD;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAsBlD;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,IAAI,IAAI,CASvC;AAED;;;GAGG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAUpD;AAED,0FAA0F;AAC1F,wBAAgB,WAAW,IAAI,IAAI,CASlC"}
|
|
@@ -52,7 +52,6 @@ exports.resetS3Sink = resetS3Sink;
|
|
|
52
52
|
const node_crypto_1 = require("node:crypto");
|
|
53
53
|
const os = __importStar(require("node:os"));
|
|
54
54
|
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
55
|
-
const date_fns_1 = require("date-fns");
|
|
56
55
|
const config_1 = require("../../config");
|
|
57
56
|
const errors_1 = require("../../lib/errors");
|
|
58
57
|
const logging_1 = require("../../lib/logging");
|
|
@@ -113,7 +112,7 @@ function bufferSubmissionLine(line) {
|
|
|
113
112
|
}
|
|
114
113
|
/** Builds the S3 object key for one uploaded batch, partitioned by day for lifecycle policies and manual browsing. */
|
|
115
114
|
function buildObjectKey(name, prefix) {
|
|
116
|
-
const datePartition =
|
|
115
|
+
const datePartition = new Date().toISOString().slice(0, 10);
|
|
117
116
|
const hostname = os.hostname();
|
|
118
117
|
const epochMs = Date.now();
|
|
119
118
|
const shortUuid = (0, node_crypto_1.randomUUID)().slice(0, 8);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3-sink.js","sourceRoot":"","sources":["../../../src/lib/telemetry/s3-sink.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,6CAAyC;AACzC,MAAY,EAAE,oCAAgB;AAE9B,kDAAgE;
|
|
1
|
+
{"version":3,"file":"s3-sink.js","sourceRoot":"","sources":["../../../src/lib/telemetry/s3-sink.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,6CAAyC;AACzC,MAAY,EAAE,oCAAgB;AAE9B,kDAAgE;AAEhE,qCAAkC;AAClC,yCAA8C;AAC9C,2CAA0C;AAE1C,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC,CAAC;AAExD,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,yBAAyB,GAAG,MAAM,CAAC;AAIzC,IAAI,MAA4B,CAAC;AACjC,IAAI,WAAW,GAAa,EAAE,CAAC;AAC/B,IAAI,iBAAiB,GAAa,EAAE,CAAC;AACrC,IAAI,KAAiD,CAAC;AACtD,IAAI,aAAwC,CAAC;AAE7C,8EAA8E;AAC9E,SAAS,SAAS;IAChB,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,GAAG,IAAI,oBAAQ,CAAC,EAAE,MAAM,EAAE,eAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,iFAAiF;AACjF,SAAS,WAAW,CAAC,MAAgB,EAAE,IAAY;IACjD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClB,IAAI,MAAM,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,CAAC;IACnD,CAAC;AACH,CAAC;AAED,0FAA0F;AAC1F,SAAS,cAAc,CAAC,MAAgB,EAAE,KAAe;IACvD,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,EAAE,GAAG,MAAM,CAAC,CAAC;IACrC,OAAO,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;AAChG,CAAC;AAED,gFAAgF;AAChF,SAAS,mBAAmB,CAAC,MAAgB;IAC3C,IAAI,MAAM,CAAC,MAAM,GAAG,eAAM,CAAC,SAAS,CAAC,EAAE,CAAC,cAAc;QAAE,OAAO;IAC/D,kBAAkB,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACjC,MAAM,CAAC,IAAI,CAAC,oCAAoC,IAAA,uBAAc,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,wBAA+B,IAAY;IACzC,IAAI,CAAC,eAAM,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM;QAAE,OAAO;IACxC,WAAW,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC/B,mBAAmB,CAAC,WAAW,CAAC,CAAC;AACnC,CAAC;AAED;;;GAGG;AACH,8BAAqC,IAAY;IAC/C,IAAI,CAAC,eAAM,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM;QAAE,OAAO;IACxC,WAAW,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC;IACrC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;AACzC,CAAC;AAED,sHAAsH;AACtH,SAAS,cAAc,CAAC,IAAgB,EAAE,MAAc;IACtD,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,IAAA,wBAAU,GAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3C,OAAO,GAAG,MAAM,IAAI,IAAI,IAAI,aAAa,IAAI,QAAQ,IAAI,OAAO,IAAI,SAAS,SAAS,CAAC;AACzF,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,QAAQ,CAAC,IAAgB,EAAE,QAAkB;IAC1D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,MAAM,GAAG,eAAM,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,CAAC;IAC1C,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAEvB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,EAAE,eAAM,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QAC7D,MAAM,SAAS,EAAE,CAAC,IAAI,CACpB,IAAI,4BAAgB,CAAC;YACnB,MAAM,EAAE,MAAM;YACd,GAAG,EAAE,GAAG;YACR,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,WAAW,EAAE,sBAAsB;SACpC,CAAC,CACH,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,WAAW,QAAQ,CAAC,MAAM,IAAI,IAAI,oBAAoB,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;QACnF,OAAO,EAAE,CAAC;IACZ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CACT,qBAAqB,IAAI,wCAAwC,IAAA,uBAAc,EAAC,GAAG,CAAC,EAAE,CACvF,CAAC;QACF,OAAO,QAAQ,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;GAYG;AACH;IACE,IAAI,CAAC,eAAM,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC1D,IAAI,aAAa;QAAE,OAAO,aAAa,CAAC;IAExC,MAAM,aAAa,GAAG,WAAW,CAAC;IAClC,MAAM,mBAAmB,GAAG,iBAAiB,CAAC;IAC9C,WAAW,GAAG,EAAE,CAAC;IACjB,iBAAiB,GAAG,EAAE,CAAC;IAEvB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;QAC1B,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;QAChC,QAAQ,CAAC,aAAa,EAAE,mBAAmB,CAAC;KAC7C,CAAC;SACC,IAAI,CAAC,CAAC,CAAC,cAAc,EAAE,oBAAoB,CAAC,EAAE,EAAE;QAC/C,WAAW,GAAG,cAAc,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;QAC1D,iBAAiB,GAAG,cAAc,CAAC,iBAAiB,EAAE,oBAAoB,CAAC,CAAC;IAC9E,CAAC,CAAC;SACD,OAAO,CAAC,GAAG,EAAE;QACZ,aAAa,GAAG,SAAS,CAAC;IAC5B,CAAC,CAAC,CAAC;IACL,aAAa,GAAG,OAAO,CAAC;IACxB,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH;IACE,IAAI,CAAC,eAAM,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM;QAAE,OAAO;IACxC,IAAI,KAAK;QAAE,OAAO;IAClB,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QACvB,kBAAkB,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACjC,MAAM,CAAC,IAAI,CAAC,mCAAmC,IAAA,uBAAc,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,eAAM,CAAC,SAAS,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC;IACxC,KAAK,CAAC,KAAK,EAAE,CAAC;AAChB,CAAC;AAED;;;GAGG;AACI,KAAK;IACV,IAAI,KAAK,EAAE,CAAC;QACV,aAAa,CAAC,KAAK,CAAC,CAAC;QACrB,KAAK,GAAG,SAAS,CAAC;IACpB,CAAC;IACD,IAAI,CAAC,eAAM,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM;QAAE,OAAO;IACxC,MAAM,OAAO,CAAC,IAAI,CAAC;QACjB,kBAAkB,EAAE;QACpB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,yBAAyB,CAAC,CAAC;KAC/E,CAAC,CAAC;AACL,CAAC;AAED,0FAA0F;AAC1F;IACE,IAAI,KAAK,EAAE,CAAC;QACV,aAAa,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,GAAG,SAAS,CAAC;IACnB,WAAW,GAAG,EAAE,CAAC;IACjB,iBAAiB,GAAG,EAAE,CAAC;IACvB,KAAK,GAAG,SAAS,CAAC;IAClB,aAAa,GAAG,SAAS,CAAC;AAC5B,CAAC"}
|
|
@@ -1,23 +1,29 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Append-only NDJSON sink for dispatch submission envelopes. One line per
|
|
3
3
|
* plugin invocation captures the outcome — siteId, requestId, inbound payload,
|
|
4
|
-
* status, audit payload, error message, duration
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* status, audit payload, error message, duration, plus the opaque `joinKeys`
|
|
5
|
+
* bag and the `kind:"submit"` dimension — so downstream tooling (ETL, jq,
|
|
6
|
+
* ad-hoc queries) can answer "what did we submit for jobId X on date Y, and
|
|
7
|
+
* did it succeed?" without a database, and a plugin can join runs to its own
|
|
8
|
+
* attribution provider's report without re-parsing `inboundPayload`.
|
|
7
9
|
*
|
|
8
10
|
* Kept on a separate sink from calls.ndjson because the judge and self-heal
|
|
9
11
|
* readers Zod-parse every calls.ndjson line as an LlmCallSample; mixing record
|
|
10
12
|
* shapes would force a discriminator + filter on every consumer.
|
|
11
13
|
*/
|
|
12
|
-
import { z } from "zod/v4";
|
|
14
|
+
import type { z } from "zod/v4";
|
|
13
15
|
/**
|
|
14
|
-
* Validated shape of one submission envelope sample.
|
|
15
|
-
*
|
|
16
|
-
*
|
|
16
|
+
* Validated shape of one submission envelope sample. Alias of
|
|
17
|
+
* `submitRecordSchema` (feat-002) kept under its original export name so the
|
|
18
|
+
* five existing test files that import it are unaffected — the reconciliation
|
|
19
|
+
* schema is a strict superset (adds `kind`, `joinKeys`, both defaulted) of
|
|
20
|
+
* the shape this module used to define standalone.
|
|
17
21
|
*/
|
|
18
22
|
export declare const submissionEnvelopeSampleSchema: z.ZodObject<{
|
|
23
|
+
kind: z.ZodDefault<z.ZodLiteral<"submit">>;
|
|
19
24
|
siteId: z.ZodString;
|
|
20
25
|
requestId: z.ZodString;
|
|
26
|
+
joinKeys: z.ZodDefault<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
|
|
21
27
|
inboundPayload: z.ZodUnknown;
|
|
22
28
|
status: z.ZodEnum<{
|
|
23
29
|
error: "error";
|
|
@@ -29,13 +35,28 @@ export declare const submissionEnvelopeSampleSchema: z.ZodObject<{
|
|
|
29
35
|
ts: z.ZodString;
|
|
30
36
|
}, z.core.$strip>;
|
|
31
37
|
export type SubmissionEnvelopeSample = z.infer<typeof submissionEnvelopeSampleSchema>;
|
|
32
|
-
/**
|
|
33
|
-
|
|
38
|
+
/**
|
|
39
|
+
* Input to `captureSubmissionEnvelope` — `ts` and `kind` are derived
|
|
40
|
+
* internally so callers omit them. `joinKeys` is optional so existing call
|
|
41
|
+
* sites that don't yet resolve join keys keep compiling unchanged; an
|
|
42
|
+
* omitted value is persisted as `null`, not left undefined.
|
|
43
|
+
*/
|
|
44
|
+
export type SubmissionEnvelopeInput = Omit<SubmissionEnvelopeSample, "ts" | "kind" | "joinKeys"> & {
|
|
45
|
+
joinKeys?: Record<string, unknown> | null;
|
|
46
|
+
};
|
|
34
47
|
/** Options for `captureSubmissionEnvelope`. */
|
|
35
48
|
export interface CaptureSubmissionOptions {
|
|
36
49
|
/** Override the sink path; used in tests to avoid touching the real file. */
|
|
37
50
|
sinkPath?: string;
|
|
38
51
|
}
|
|
52
|
+
/**
|
|
53
|
+
* Ensures the sink directory exists, appends `line`, and forwards it to the
|
|
54
|
+
* S3 buffer. Extracted so the beacon-event writer can share the exact
|
|
55
|
+
* mkdir/append/buffer sequence instead of re-implementing it (CLAUDE.md
|
|
56
|
+
* §DRY) — callers own their own error handling since the swallow-all
|
|
57
|
+
* contract differs per writer.
|
|
58
|
+
*/
|
|
59
|
+
export declare function appendSubmissionSinkLine(sinkPath: string, line: string): Promise<void>;
|
|
39
60
|
/**
|
|
40
61
|
* Appends one validated NDJSON line for a dispatch outcome to the configured
|
|
41
62
|
* sink. Errors are logged and swallowed — telemetry must never break the
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"submission-capture.d.ts","sourceRoot":"","sources":["../../../src/lib/telemetry/submission-capture.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"submission-capture.d.ts","sourceRoot":"","sources":["../../../src/lib/telemetry/submission-capture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAMH,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAShC;;;;;;GAMG;AACH,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;iBAAqB,CAAC;AAEjE,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAAC;AAEtF;;;;;GAKG;AACH,MAAM,MAAM,uBAAuB,GAAG,IAAI,CAAC,wBAAwB,EAAE,IAAI,GAAG,MAAM,GAAG,UAAU,CAAC,GAAG;IACjG,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC3C,CAAC;AAEF,+CAA+C;AAC/C,MAAM,WAAW,wBAAwB;IACvC,6EAA6E;IAC7E,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;GAMG;AACH,wBAAsB,wBAAwB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAI5F;AAED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,KAAK,EAAE,uBAAuB,EAC9B,IAAI,GAAE,wBAA6B,GAClC,OAAO,CAAC,IAAI,CAAC,CAef"}
|
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* Append-only NDJSON sink for dispatch submission envelopes. One line per
|
|
4
4
|
* plugin invocation captures the outcome — siteId, requestId, inbound payload,
|
|
5
|
-
* status, audit payload, error message, duration
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* status, audit payload, error message, duration, plus the opaque `joinKeys`
|
|
6
|
+
* bag and the `kind:"submit"` dimension — so downstream tooling (ETL, jq,
|
|
7
|
+
* ad-hoc queries) can answer "what did we submit for jobId X on date Y, and
|
|
8
|
+
* did it succeed?" without a database, and a plugin can join runs to its own
|
|
9
|
+
* attribution provider's report without re-parsing `inboundPayload`.
|
|
8
10
|
*
|
|
9
11
|
* Kept on a separate sink from calls.ndjson because the judge and self-heal
|
|
10
12
|
* readers Zod-parse every calls.ndjson line as an LlmCallSample; mixing record
|
|
@@ -12,30 +14,36 @@
|
|
|
12
14
|
*/
|
|
13
15
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
16
|
exports.submissionEnvelopeSampleSchema = void 0;
|
|
17
|
+
exports.appendSubmissionSinkLine = appendSubmissionSinkLine;
|
|
15
18
|
exports.captureSubmissionEnvelope = captureSubmissionEnvelope;
|
|
16
19
|
const promises_1 = require("node:fs/promises");
|
|
17
20
|
const node_path_1 = require("node:path");
|
|
18
21
|
const date_fns_1 = require("date-fns");
|
|
19
|
-
const v4_1 = require("zod/v4");
|
|
20
22
|
const config_1 = require("../../config");
|
|
21
23
|
const logging_1 = require("../../lib/logging");
|
|
24
|
+
const reconciliation_record_1 = require("../../lib/telemetry/reconciliation-record");
|
|
22
25
|
const s3_sink_1 = require("../../lib/telemetry/s3-sink");
|
|
23
26
|
const logger = (0, logging_1.getLogger)({ name: "telemetry/submission-capture" });
|
|
24
27
|
/**
|
|
25
|
-
* Validated shape of one submission envelope sample.
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
+
* Validated shape of one submission envelope sample. Alias of
|
|
29
|
+
* `submitRecordSchema` (feat-002) kept under its original export name so the
|
|
30
|
+
* five existing test files that import it are unaffected — the reconciliation
|
|
31
|
+
* schema is a strict superset (adds `kind`, `joinKeys`, both defaulted) of
|
|
32
|
+
* the shape this module used to define standalone.
|
|
28
33
|
*/
|
|
29
|
-
exports.submissionEnvelopeSampleSchema =
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
});
|
|
34
|
+
exports.submissionEnvelopeSampleSchema = reconciliation_record_1.submitRecordSchema;
|
|
35
|
+
/**
|
|
36
|
+
* Ensures the sink directory exists, appends `line`, and forwards it to the
|
|
37
|
+
* S3 buffer. Extracted so the beacon-event writer can share the exact
|
|
38
|
+
* mkdir/append/buffer sequence instead of re-implementing it (CLAUDE.md
|
|
39
|
+
* §DRY) — callers own their own error handling since the swallow-all
|
|
40
|
+
* contract differs per writer.
|
|
41
|
+
*/
|
|
42
|
+
async function appendSubmissionSinkLine(sinkPath, line) {
|
|
43
|
+
await (0, promises_1.mkdir)((0, node_path_1.dirname)(sinkPath), { recursive: true });
|
|
44
|
+
await (0, promises_1.appendFile)(sinkPath, line, "utf8");
|
|
45
|
+
(0, s3_sink_1.bufferSubmissionLine)(line);
|
|
46
|
+
}
|
|
39
47
|
/**
|
|
40
48
|
* Appends one validated NDJSON line for a dispatch outcome to the configured
|
|
41
49
|
* sink. Errors are logged and swallowed — telemetry must never break the
|
|
@@ -43,12 +51,15 @@ exports.submissionEnvelopeSampleSchema = v4_1.z.object({
|
|
|
43
51
|
*/
|
|
44
52
|
async function captureSubmissionEnvelope(input, opts = {}) {
|
|
45
53
|
const sinkPath = opts.sinkPath ?? config_1.config.telemetry.submissionsNdjsonPath;
|
|
46
|
-
const sample = {
|
|
54
|
+
const sample = {
|
|
55
|
+
...input,
|
|
56
|
+
kind: "submit",
|
|
57
|
+
joinKeys: input.joinKeys ?? null,
|
|
58
|
+
ts: (0, date_fns_1.formatISO)(new Date()),
|
|
59
|
+
};
|
|
47
60
|
try {
|
|
48
61
|
const line = `${JSON.stringify(sample)}\n`;
|
|
49
|
-
await (
|
|
50
|
-
await (0, promises_1.appendFile)(sinkPath, line, "utf8");
|
|
51
|
-
(0, s3_sink_1.bufferSubmissionLine)(line);
|
|
62
|
+
await appendSubmissionSinkLine(sinkPath, line);
|
|
52
63
|
}
|
|
53
64
|
catch (err) {
|
|
54
65
|
logger.error(`captureSubmissionEnvelope: failed to write to ${sinkPath}: ${String(err)}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"submission-capture.js","sourceRoot":"","sources":["../../../src/lib/telemetry/submission-capture.ts"],"names":[],"mappings":";AAAA
|
|
1
|
+
{"version":3,"file":"submission-capture.js","sourceRoot":"","sources":["../../../src/lib/telemetry/submission-capture.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;GAYG;;;;;AAEH,+CAAqD;AACrD,yCAAoC;AAEpC,uCAAqC;AAGrC,qCAAkC;AAClC,2CAA0C;AAC1C,iFAA2E;AAC3E,qDAA+D;AAE/D,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,8BAA8B,EAAE,CAAC,CAAC;AAEnE;;;;;;GAMG;AACU,QAAA,8BAA8B,GAAG,0CAAkB,CAAC;AAoBjE;;;;;;GAMG;AACI,KAAK,mCAAmC,QAAgB,EAAE,IAAY;IAC3E,MAAM,IAAA,gBAAK,EAAC,IAAA,mBAAO,EAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,MAAM,IAAA,qBAAU,EAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACzC,IAAA,8BAAoB,EAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED;;;;GAIG;AACI,KAAK,oCACV,KAA8B,EAC9B,IAAI,GAA6B,EAAE;IAEnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAM,CAAC,SAAS,CAAC,qBAAqB,CAAC;IACzE,MAAM,MAAM,GAA6B;QACvC,GAAG,KAAK;QACR,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;QAChC,EAAE,EAAE,IAAA,oBAAS,EAAC,IAAI,IAAI,EAAE,CAAC;KAC1B,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC;QAC3C,MAAM,wBAAwB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,iDAAiD,QAAQ,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC5F,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Predicate composition over `ReconciliationRow[]` (feat-007's reader
|
|
3
|
+
* output), so a caller can ask reconciliation-shaped questions — rows for a
|
|
4
|
+
* given `siteId` cohort, `requestId`, submit outcome, beacon status, or date
|
|
5
|
+
* window — without re-parsing the submissions NDJSON. Pure and I/O-free: the
|
|
6
|
+
* route layer composes `readDurableReconciliationRows`
|
|
7
|
+
* (`src/lib/telemetry/reconciliation-source.ts`) then this module.
|
|
8
|
+
*/
|
|
9
|
+
import type { SubmitRecord } from "../../lib/telemetry/reconciliation-record";
|
|
10
|
+
import type { ReconciliationRow } from "../../lib/telemetry/submission-reader";
|
|
11
|
+
/**
|
|
12
|
+
* Filters for `queryReconciliationRows`. Every field is optional; the
|
|
13
|
+
* fields present are combined with AND. `from`/`to` bound the `ts` window
|
|
14
|
+
* inclusively; `limit`/`offset` paginate the newest-first result.
|
|
15
|
+
*/
|
|
16
|
+
export interface ReconciliationRowFilter {
|
|
17
|
+
siteId?: string;
|
|
18
|
+
requestId?: string;
|
|
19
|
+
status?: SubmitRecord["status"];
|
|
20
|
+
beaconStatus?: ReconciliationRow["beaconStatus"];
|
|
21
|
+
from?: string;
|
|
22
|
+
to?: string;
|
|
23
|
+
limit?: number;
|
|
24
|
+
offset?: number;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Filters, sorts (newest-first by `ts`), and paginates reconciliation rows
|
|
28
|
+
* so the attribution-report join a human pulls in pages is deterministic
|
|
29
|
+
* rather than relying on the sink's append order.
|
|
30
|
+
*/
|
|
31
|
+
export declare function queryReconciliationRows(rows: ReconciliationRow[], filter?: ReconciliationRowFilter): ReconciliationRow[];
|
|
32
|
+
//# sourceMappingURL=submission-query.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"submission-query.d.ts","sourceRoot":"","sources":["../../../src/lib/telemetry/submission-query.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uCAAuC,CAAC;AAC1E,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAC;AAE3E;;;;GAIG;AACH,MAAM,WAAW,uBAAuB;IACtC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChC,YAAY,CAAC,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IACjD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AA2BD;;;;GAIG;AACH,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,iBAAiB,EAAE,EACzB,MAAM,GAAE,uBAA4B,GACnC,iBAAiB,EAAE,CAWrB"}
|