@usherlabs/cex-broker 0.2.45 → 0.2.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +1 -1
  2. package/dist/commands/cex-canonical-orderbook-export.d.ts +23 -0
  3. package/dist/commands/cex-canonical-orderbook-export.js +27731 -0
  4. package/dist/commands/cex-canonical-orderbook-export.js.map +247 -0
  5. package/dist/commands/cli.js +16231 -15573
  6. package/dist/commands/market-data-vendor-backfill.d.ts +24 -0
  7. package/dist/commands/market-data-vendor-backfill.js +33316 -0
  8. package/dist/commands/market-data-vendor-backfill.js.map +279 -0
  9. package/dist/handlers/subscribe/handler.d.ts +2 -0
  10. package/dist/helpers/canonical-orderbook-export/exact-selection.d.ts +21 -0
  11. package/dist/helpers/canonical-orderbook-export/exporter.d.ts +26 -0
  12. package/dist/helpers/market-data-archive/capture-contract.d.ts +3 -2
  13. package/dist/helpers/market-data-archive/types.d.ts +8 -4
  14. package/dist/helpers/market-data-preparation/conformance-fixtures.d.ts +16 -0
  15. package/dist/helpers/market-data-preparation/contracts.d.ts +1690 -0
  16. package/dist/helpers/market-data-preparation/file-job.d.ts +38 -0
  17. package/dist/helpers/market-data-vendor-backfill/archive-reader.d.ts +26 -0
  18. package/dist/helpers/market-data-vendor-backfill/batching.d.ts +10 -0
  19. package/dist/helpers/market-data-vendor-backfill/conformance-fixtures.d.ts +80 -0
  20. package/dist/helpers/market-data-vendor-backfill/contracts.d.ts +393 -0
  21. package/dist/helpers/market-data-vendor-backfill/core.d.ts +100 -0
  22. package/dist/helpers/market-data-vendor-backfill/cryptohftdata.d.ts +77 -0
  23. package/dist/helpers/market-data-vendor-backfill/forwarder-client.d.ts +25 -0
  24. package/dist/helpers/market-data-vendor-backfill/identity.d.ts +10 -0
  25. package/dist/helpers/market-data-vendor-backfill/manifests.d.ts +1086 -0
  26. package/dist/helpers/market-data-vendor-backfill/promotion.d.ts +9 -0
  27. package/dist/helpers/market-data-vendor-backfill/qualification.d.ts +20 -0
  28. package/dist/helpers/market-data-vendor-backfill/selection.d.ts +30 -0
  29. package/dist/helpers/market-data-vendor-backfill/semantic-verification.d.ts +24 -0
  30. package/dist/helpers/public-market-data-feed/identity.d.ts +12 -0
  31. package/dist/helpers/public-market-data-feed/index.d.ts +5 -0
  32. package/dist/helpers/public-market-data-feed/orderbook-coalescing-evidence.d.ts +79 -0
  33. package/dist/helpers/public-market-data-feed/orderbook-profile.d.ts +54 -0
  34. package/dist/helpers/public-market-data-feed/subscriber-buffer.d.ts +31 -0
  35. package/dist/helpers/public-market-data-feed/supervisor.d.ts +82 -0
  36. package/dist/index.d.ts +1 -0
  37. package/dist/index.js +16221 -15563
  38. package/dist/index.js.map +21 -18
  39. package/dist/market-data-preparation/fixtures/conformance-v2.json +424 -0
  40. package/dist/market-data-preparation/policies/capability-policy.json +28 -0
  41. package/dist/market-data-preparation/policies/resource-policy.json +16 -0
  42. package/dist/market-data-preparation/schema-manifest.json +46 -0
  43. package/dist/market-data-preparation/schemas/archive-selection-v1.schema.json +224 -0
  44. package/dist/market-data-preparation/schemas/backfill-request-v1.schema.json +166 -0
  45. package/dist/market-data-preparation/schemas/backfill-result-v2.schema.json +212 -0
  46. package/dist/market-data-preparation/schemas/canonical-orderbook-export-request-v1.schema.json +57 -0
  47. package/dist/market-data-preparation/schemas/canonical-orderbook-export-result-v1.schema.json +238 -0
  48. package/dist/market-data-preparation/schemas/preparation-product-pin-v1.schema.json +126 -0
  49. package/dist/market-data-preparation/schemas/promotion-receipt-v1.schema.json +171 -0
  50. package/dist/market-data-preparation/schemas/required-clock-v1.schema.json +47 -0
  51. package/dist/market-data-preparation.d.ts +4 -0
  52. package/dist/market-data-preparation.js +27569 -0
  53. package/dist/market-data-preparation.js.map +249 -0
  54. package/dist/market-data-vendor-backfill/fixtures/conformance-v1.json +448 -0
  55. package/dist/market-data-vendor-backfill/policies/capability-policy.json +28 -0
  56. package/dist/market-data-vendor-backfill/policies/resource-policy.json +16 -0
  57. package/dist/market-data-vendor-backfill/schema-manifest.json +31 -0
  58. package/dist/market-data-vendor-backfill/schemas/archive-selection.schema.json +224 -0
  59. package/dist/market-data-vendor-backfill/schemas/promotion-receipt.schema.json +171 -0
  60. package/dist/market-data-vendor-backfill/schemas/request.schema.json +166 -0
  61. package/dist/market-data-vendor-backfill/schemas/required-clock.schema.json +47 -0
  62. package/dist/market-data-vendor-backfill/schemas/result.schema.json +198 -0
  63. package/dist/market-data-vendor-backfill.d.ts +18 -0
  64. package/dist/market-data-vendor-backfill.js +18850 -0
  65. package/dist/market-data-vendor-backfill.js.map +206 -0
  66. package/dist/server.d.ts +2 -1
  67. package/package.json +37 -3
@@ -0,0 +1,38 @@
1
+ export declare const FILE_JOB_REQUEST_MAX_BYTES = 1048576;
2
+ export declare const FILE_JOB_CLOCK_MAX_BYTES = 16777216;
3
+ export declare const FILE_JOB_EXECUTABLE_MAX_BYTES = 536870912;
4
+ export type FileJobPaths = {
5
+ attemptRoot: string;
6
+ requestPath: string;
7
+ resultPath: string;
8
+ };
9
+ export type AtomicWriteHooks = {
10
+ beforeTempOpen?(tempPath: string): void | Promise<void>;
11
+ beforeFileFsync?(): void | Promise<void>;
12
+ beforeRename?(): void | Promise<void>;
13
+ beforeDirectoryFsync?(): void | Promise<void>;
14
+ };
15
+ export type AtomicWriteJsonOptions<T> = {
16
+ validate?: (value: T) => void;
17
+ hooks?: AtomicWriteHooks;
18
+ };
19
+ export type DurableArtifactHooks = {
20
+ beforeOpen?(filePath: string): void | Promise<void>;
21
+ beforeFileFsync?(): void | Promise<void>;
22
+ };
23
+ export declare class FileJobUnreadableError extends Error {
24
+ constructor(message: string);
25
+ }
26
+ export declare function containsParentTraversal(input: string): boolean;
27
+ export declare function assertSupportedNode22(version?: string, productName?: string): void;
28
+ export declare function parseFileJobArgv(argv: readonly string[]): {
29
+ requestPath: string;
30
+ resultPath: string;
31
+ };
32
+ export declare function assertSafeFileJobPaths(requestInput: string, resultInput: string): Promise<FileJobPaths>;
33
+ export declare function assertSidecarBasename(fileName: string): void;
34
+ export declare function readBoundedRegularFile(filePath: string, maximumBytes: number): Promise<Buffer>;
35
+ export declare function sha256RegularFile(filePath: string): Promise<string>;
36
+ export declare function writeExclusiveDurableFile(filePath: string, bytes: Uint8Array, hooks?: DurableArtifactHooks): Promise<void>;
37
+ export declare function redactDiagnostics(diagnostics: Readonly<Record<string, string | number | boolean>>, sensitiveValues: ReadonlySet<string>): Record<string, string | number | boolean>;
38
+ export declare function atomicWriteJsonResult<T>(resultPath: string, result: T, options?: AtomicWriteJsonOptions<T>): Promise<void>;
@@ -0,0 +1,26 @@
1
+ import type { MarketDataVendorBackfillRequest } from "./contracts";
2
+ import type { ArchivePreflightResolution, CandidateVerification, NormalizedBackfill, QualifiedCoverage } from "./core";
3
+ export type ArchiveQueryValue = string | number | readonly string[];
4
+ export type ArchiveQueryClient = {
5
+ query(sql: string, parameters?: Readonly<Record<string, ArchiveQueryValue>>): Promise<Record<string, unknown>[]>;
6
+ };
7
+ export type QualifiedArchiveReaderOptions = {
8
+ nowMs?: () => number;
9
+ };
10
+ export declare class QualifiedOrderBookArchiveReader {
11
+ private readonly client;
12
+ private readonly options?;
13
+ constructor(client: ArchiveQueryClient, options?: QualifiedArchiveReaderOptions | undefined);
14
+ resolveSelection(request: MarketDataVendorBackfillRequest): Promise<ArchivePreflightResolution>;
15
+ private readReceipts;
16
+ coverage(request: MarketDataVendorBackfillRequest): Promise<QualifiedCoverage>;
17
+ private boundaryRows;
18
+ verifyCandidate(request: MarketDataVendorBackfillRequest, normalized: NormalizedBackfill, captureBundleId: string, baseline: ArchivePreflightResolution): Promise<CandidateVerification>;
19
+ private seamIsOrdered;
20
+ }
21
+ export declare function createClickHouseArchiveQueryClient(input: {
22
+ url: string;
23
+ username?: string;
24
+ password?: string;
25
+ fetch?: typeof fetch;
26
+ }): ArchiveQueryClient;
@@ -0,0 +1,10 @@
1
+ import { type BackfillArchiveRow, type ForwarderBatch } from "./contracts";
2
+ export declare const BACKFILL_MAX_BATCH_ROWS = 1000;
3
+ export declare const BACKFILL_MAX_BATCH_BYTES: number;
4
+ export declare function buildForwarderBatches(input: {
5
+ captureBundleId: string;
6
+ deploymentId: string;
7
+ rows: readonly BackfillArchiveRow[];
8
+ maxRows?: number;
9
+ maxBytes?: number;
10
+ }): ForwarderBatch[];
@@ -0,0 +1,80 @@
1
+ export declare const CONFORMANCE_FIXTURES: Readonly<{
2
+ fixture_id: "market-data-vendor-backfill-conformance/v1";
3
+ documents: {
4
+ required_clock: import("./contracts").RequiredClockWire;
5
+ archive_selection: import("./contracts").ArchiveSelectionWire;
6
+ request: {
7
+ idempotency_key: string;
8
+ schema_id: string;
9
+ request_id: string;
10
+ attempt_id: string;
11
+ scope: {
12
+ exchange: string;
13
+ trading_pair: string;
14
+ market_type: "spot";
15
+ feed: "ORDERBOOK";
16
+ };
17
+ window: {
18
+ start_at: string;
19
+ end_at: string;
20
+ };
21
+ depth: number;
22
+ construction_mode: "sampled_top_n_snapshot";
23
+ source_policy: "authoritative_window";
24
+ target: {
25
+ environment: string;
26
+ cluster: string;
27
+ };
28
+ coverage_policy: {
29
+ policy_id: "prior-asof-strict/v1";
30
+ max_asof_lag_ms: number;
31
+ future_rows: "reject";
32
+ missing_required_event: "fail";
33
+ };
34
+ required_clock: {
35
+ schema_id: string;
36
+ clock_id: string;
37
+ file_name: string;
38
+ clock_sha256: string;
39
+ event_count: number;
40
+ };
41
+ initial_selection: import("./contracts").ArchiveSelectionWire;
42
+ expected_canonical_schema: {
43
+ schema_id: string;
44
+ schema_sha256: string;
45
+ };
46
+ product_pins: {
47
+ capability_policy: {
48
+ policy_id: "market-data-vendor-backfill-capabilities/v1";
49
+ policy_sha256: string;
50
+ };
51
+ resource_policy: {
52
+ policy_id: "market-data-vendor-backfill-resources/v1";
53
+ policy_sha256: string;
54
+ };
55
+ };
56
+ production_authorization_id: string;
57
+ };
58
+ promotion_receipt: import("./contracts").PromotionReceiptWire;
59
+ result: import("./contracts").BackfillJobResultWire;
60
+ };
61
+ identities: {
62
+ idempotency_key: string;
63
+ clock_sha256: string;
64
+ initial_selection_sha256: string;
65
+ selection_sha256: string;
66
+ promotion_identity_sha256: string;
67
+ receipt_id: string;
68
+ result_sha256: string;
69
+ capability_policy_sha256: string;
70
+ resource_policy_sha256: string;
71
+ };
72
+ jcs_edge_vector: {
73
+ literals: (boolean | null)[];
74
+ numbers: number[];
75
+ string: string;
76
+ };
77
+ hashes: {
78
+ jcs_edge_vector_sha256: string;
79
+ };
80
+ }>;
@@ -0,0 +1,393 @@
1
+ import { z } from "zod";
2
+ export declare const BACKFILL_REQUEST_SCHEMA_ID: string;
3
+ export declare const REQUIRED_CLOCK_SCHEMA_ID: string;
4
+ export declare const ARCHIVE_SELECTION_SCHEMA_ID: string;
5
+ export declare const PROMOTION_RECEIPT_SCHEMA_ID: string;
6
+ export declare const BACKFILL_RESULT_SCHEMA_ID: string;
7
+ export type FixedUtcTimestamp = string;
8
+ export type Sha256Hex = string;
9
+ export type LowercaseUuid = string;
10
+ export type CanonicalScopeWire = {
11
+ exchange: string;
12
+ trading_pair: string;
13
+ market_type: "spot" | "swap" | "future";
14
+ feed: "ORDERBOOK";
15
+ };
16
+ export type CoveragePolicyWire = {
17
+ policy_id: "prior-asof-strict/v1";
18
+ max_asof_lag_ms: number;
19
+ future_rows: "reject";
20
+ missing_required_event: "fail";
21
+ };
22
+ export type RequiredClockWire = {
23
+ schema_id: typeof REQUIRED_CLOCK_SCHEMA_ID;
24
+ clock_id: LowercaseUuid;
25
+ clock_sha256: Sha256Hex;
26
+ created_at: FixedUtcTimestamp;
27
+ targets: Array<{
28
+ target_id: LowercaseUuid;
29
+ target_at: FixedUtcTimestamp;
30
+ }>;
31
+ };
32
+ export type ArchiveSelectionWire = {
33
+ schema_id: typeof ARCHIVE_SELECTION_SCHEMA_ID;
34
+ selection_sha256: Sha256Hex;
35
+ scope: CanonicalScopeWire;
36
+ required_clock: {
37
+ clock_id: LowercaseUuid;
38
+ clock_sha256: Sha256Hex;
39
+ event_count: number;
40
+ };
41
+ coverage_policy: CoveragePolicyWire;
42
+ source_policy: "authoritative_window" | "fill_gaps";
43
+ coverage_class: "complete" | "partial" | "missing";
44
+ requested_intervals: Array<{
45
+ start_at: FixedUtcTimestamp;
46
+ end_at: FixedUtcTimestamp;
47
+ }>;
48
+ selected_intervals: Array<{
49
+ start_at: FixedUtcTimestamp;
50
+ end_at: FixedUtcTimestamp;
51
+ capture_bundle_id: Sha256Hex;
52
+ capture_origin: "production_capture" | "vendor_historical_backfill";
53
+ }>;
54
+ precedence: Array<"archive" | "vendor">;
55
+ bundles: Array<{
56
+ capture_bundle_id: Sha256Hex;
57
+ capture_origin: "production_capture" | "vendor_historical_backfill";
58
+ interval: {
59
+ start_at: FixedUtcTimestamp;
60
+ end_at: FixedUtcTimestamp;
61
+ };
62
+ qualification: null | {
63
+ qualification_event_id: LowercaseUuid;
64
+ state: "qualified";
65
+ receipt_id: Sha256Hex;
66
+ promotion_identity_sha256: Sha256Hex;
67
+ };
68
+ }>;
69
+ support_anchors: Array<{
70
+ capture_bundle_id: Sha256Hex;
71
+ raw_capture_id: Sha256Hex;
72
+ snapshot_id: Sha256Hex;
73
+ source_time: FixedUtcTimestamp;
74
+ normalized_summary_checksum: Sha256Hex;
75
+ metadata_ref: {
76
+ capture_origin: "production_capture" | "vendor_historical_backfill";
77
+ qualification_event_id: LowercaseUuid | null;
78
+ receipt_id: Sha256Hex | null;
79
+ };
80
+ }>;
81
+ receipt_ids: Sha256Hex[];
82
+ qualification_event_ids: LowercaseUuid[];
83
+ resolved_at: FixedUtcTimestamp;
84
+ };
85
+ export type BackfillRequestWire = {
86
+ schema_id: typeof BACKFILL_REQUEST_SCHEMA_ID;
87
+ request_id: LowercaseUuid;
88
+ attempt_id: LowercaseUuid;
89
+ idempotency_key: Sha256Hex;
90
+ scope: CanonicalScopeWire;
91
+ window: {
92
+ start_at: FixedUtcTimestamp;
93
+ end_at: FixedUtcTimestamp;
94
+ };
95
+ depth: number;
96
+ construction_mode: "sampled_top_n_snapshot" | "exact_l2_reconstruction";
97
+ source_policy: "authoritative_window" | "fill_gaps";
98
+ target: {
99
+ environment: string;
100
+ cluster: string;
101
+ };
102
+ coverage_policy: CoveragePolicyWire;
103
+ required_clock: {
104
+ schema_id: typeof REQUIRED_CLOCK_SCHEMA_ID;
105
+ clock_id: LowercaseUuid;
106
+ file_name: string;
107
+ clock_sha256: Sha256Hex;
108
+ event_count: number;
109
+ };
110
+ initial_selection: ArchiveSelectionWire;
111
+ expected_canonical_schema: {
112
+ schema_id: string;
113
+ schema_sha256: Sha256Hex;
114
+ };
115
+ product_pins: {
116
+ capability_policy: {
117
+ policy_id: string;
118
+ policy_sha256: Sha256Hex;
119
+ };
120
+ resource_policy: {
121
+ policy_id: string;
122
+ policy_sha256: Sha256Hex;
123
+ };
124
+ };
125
+ production_authorization_id: LowercaseUuid;
126
+ };
127
+ export type PromotionReceiptWire = {
128
+ schema_id: typeof PROMOTION_RECEIPT_SCHEMA_ID;
129
+ receipt_id: Sha256Hex;
130
+ promotion_identity_sha256: Sha256Hex;
131
+ verified_at: FixedUtcTimestamp;
132
+ request_id: LowercaseUuid;
133
+ idempotency_key: Sha256Hex;
134
+ source: "external_backfill";
135
+ capture_origin: "vendor_historical_backfill";
136
+ source_mode: "vendor_historical_backfill_v1";
137
+ provider: string;
138
+ adapter_version: string;
139
+ effective_policies: {
140
+ capability_policy: {
141
+ policy_id: string;
142
+ policy_sha256: Sha256Hex;
143
+ };
144
+ resource_policy: {
145
+ policy_id: string;
146
+ policy_sha256: Sha256Hex;
147
+ };
148
+ adapter_policy: {
149
+ policy_id: string;
150
+ policy_sha256: Sha256Hex;
151
+ };
152
+ acquisition_policy: {
153
+ policy_id: string;
154
+ policy_sha256: Sha256Hex;
155
+ };
156
+ };
157
+ capture_bundle_id: Sha256Hex;
158
+ scope: CanonicalScopeWire;
159
+ window: {
160
+ start_at: FixedUtcTimestamp;
161
+ end_at: FixedUtcTimestamp;
162
+ };
163
+ depth: number;
164
+ construction_mode: "sampled_top_n_snapshot" | "exact_l2_reconstruction";
165
+ canonical_schema: {
166
+ schema_id: string;
167
+ schema_sha256: Sha256Hex;
168
+ };
169
+ coverage_policy: CoveragePolicyWire;
170
+ selection_sha256: Sha256Hex;
171
+ vendor_semantic_digest: Sha256Hex;
172
+ canonical_semantic_digest: Sha256Hex;
173
+ prefix_digest: Sha256Hex;
174
+ suffix_digest: Sha256Hex;
175
+ seam_verified: true;
176
+ coverage_verified: true;
177
+ dataset_objects: Array<{
178
+ identity: string;
179
+ checksum: Sha256Hex;
180
+ bytes: number;
181
+ rows: number;
182
+ }>;
183
+ };
184
+ export declare const FINAL_BACKFILL_STATUSES: readonly ["request_invalid", "archive_preflight_failed", "already_covered", "promoted", "capability_unsupported", "credentials_missing", "vendor_fetch_failed", "archive_ingest_failed", "promotion_verification_failed"];
185
+ export type FinalBackfillStatus = (typeof FINAL_BACKFILL_STATUSES)[number];
186
+ export type BackfillJobResultWire = {
187
+ schema_id: typeof BACKFILL_RESULT_SCHEMA_ID;
188
+ result_sha256: Sha256Hex;
189
+ job_id: LowercaseUuid;
190
+ request_file_sha256: Sha256Hex | null;
191
+ executable_sha256: Sha256Hex;
192
+ schema_manifest_sha256: Sha256Hex;
193
+ cex_package: {
194
+ name: "@usherlabs/cex-broker";
195
+ version: string;
196
+ package_sha256: Sha256Hex;
197
+ };
198
+ capability_policy: {
199
+ policy_id: string;
200
+ policy_sha256: Sha256Hex;
201
+ };
202
+ resource_policy: {
203
+ policy_id: string;
204
+ policy_sha256: Sha256Hex;
205
+ };
206
+ build: {
207
+ fiet_tee_commit: string;
208
+ created_at: FixedUtcTimestamp;
209
+ };
210
+ started_at: FixedUtcTimestamp;
211
+ completed_at: FixedUtcTimestamp;
212
+ outcome: {
213
+ status: FinalBackfillStatus;
214
+ reason_code: string;
215
+ reason_subcode: string | null;
216
+ request_id: LowercaseUuid | null;
217
+ idempotency_key: Sha256Hex | null;
218
+ target: {
219
+ environment: string;
220
+ cluster: string;
221
+ } | null;
222
+ selection: ArchiveSelectionWire | null;
223
+ receipt: PromotionReceiptWire | null;
224
+ diagnostics: Record<string, string | number | boolean>;
225
+ };
226
+ };
227
+ type Codec<T> = {
228
+ decode(value: unknown): T;
229
+ is(value: unknown): value is T;
230
+ };
231
+ export declare const requiredClockCodec: Codec<RequiredClockWire>;
232
+ export declare const archiveSelectionCodec: Codec<ArchiveSelectionWire>;
233
+ export declare function promotionIdentitySha256(receipt: Omit<PromotionReceiptWire, "receipt_id" | "promotion_identity_sha256"> & Partial<Pick<PromotionReceiptWire, "receipt_id" | "promotion_identity_sha256">>): string;
234
+ export declare const promotionReceiptCodec: Codec<PromotionReceiptWire>;
235
+ export declare const backfillResultCodec: Codec<BackfillJobResultWire>;
236
+ export declare const backfillRequestCodec: {
237
+ decode(value: unknown): {
238
+ wire: BackfillRequestWire;
239
+ };
240
+ is(value: unknown): value is BackfillRequestWire;
241
+ };
242
+ export declare function finalizeRequiredClock(clock: Omit<RequiredClockWire, "clock_sha256">): RequiredClockWire;
243
+ export declare function finalizeArchiveSelection(selection: Omit<ArchiveSelectionWire, "selection_sha256">): ArchiveSelectionWire;
244
+ export declare function finalizeBackfillResult(result: Omit<BackfillJobResultWire, "result_sha256"> & Partial<Pick<BackfillJobResultWire, "result_sha256">>): BackfillJobResultWire;
245
+ export declare const BACKFILL_REQUEST_SCHEMA_VERSION: "market-data-vendor-backfill-request/v1";
246
+ export declare const BACKFILL_RESULT_SCHEMA_VERSION: "market-data-vendor-backfill-result/v1";
247
+ export declare const BACKFILL_PROMOTION_SCHEMA_VERSION: "market-data-vendor-backfill-promotion-receipt/v1";
248
+ export declare const EXTERNAL_BACKFILL_SOURCE: "external_backfill";
249
+ export declare const HISTORICAL_VENDOR_SOURCE_MODE: "vendor_historical_backfill_v1";
250
+ export declare const VENDOR_DATASET_RAW_CAPTURE_SCOPE: "vendor_normalized_dataset_file";
251
+ export declare const backfillRequestSchema: z.ZodObject<{
252
+ schemaVersion: z.ZodLiteral<"market-data-vendor-backfill-request/v1">;
253
+ requestId: z.ZodString;
254
+ idempotencyKey: z.ZodString;
255
+ providerPolicy: z.ZodObject<{
256
+ provider: z.ZodLiteral<"cryptohftdata">;
257
+ allowedAdapterVersions: z.ZodArray<z.ZodString>;
258
+ }, z.core.$strict>;
259
+ scope: z.ZodObject<{
260
+ exchange: z.ZodString;
261
+ tradingPair: z.ZodString;
262
+ sourceSymbol: z.ZodString;
263
+ marketType: z.ZodEnum<{
264
+ spot: "spot";
265
+ swap: "swap";
266
+ future: "future";
267
+ }>;
268
+ feed: z.ZodLiteral<"ORDERBOOK">;
269
+ }, z.core.$strict>;
270
+ window: z.ZodObject<{
271
+ startTimeMs: z.ZodNumber;
272
+ endTimeMs: z.ZodNumber;
273
+ }, z.core.$strict>;
274
+ depth: z.ZodNumber;
275
+ constructionMode: z.ZodEnum<{
276
+ sampled_top_n_snapshot: "sampled_top_n_snapshot";
277
+ exact_l2_reconstruction: "exact_l2_reconstruction";
278
+ }>;
279
+ requiredClockTargetsMs: z.ZodArray<z.ZodNumber>;
280
+ maxPriorAsOfLagMs: z.ZodNumber;
281
+ sourcePolicy: z.ZodEnum<{
282
+ authoritative_window: "authoritative_window";
283
+ fill_gaps: "fill_gaps";
284
+ }>;
285
+ budgets: z.ZodObject<{
286
+ maxFiles: z.ZodNumber;
287
+ maxBytes: z.ZodNumber;
288
+ maxRows: z.ZodNumber;
289
+ maxDurationMs: z.ZodNumber;
290
+ maxBoundaryLookbackMs: z.ZodNumber;
291
+ }, z.core.$strict>;
292
+ expectedProduct: z.ZodObject<{
293
+ packageName: z.ZodLiteral<"@usherlabs/cex-broker">;
294
+ packageVersion: z.ZodOptional<z.ZodString>;
295
+ canonicalSchemaVersion: z.ZodString;
296
+ checksumAlgorithm: z.ZodLiteral<"sha256-canonical-json-v1">;
297
+ }, z.core.$strict>;
298
+ }, z.core.$strict>;
299
+ type ProvisionalBackfillDomainRequest = z.infer<typeof backfillRequestSchema>;
300
+ export type MarketDataVendorBackfillRequest = ProvisionalBackfillDomainRequest & {
301
+ attemptId?: string;
302
+ target?: {
303
+ environment: string;
304
+ cluster: string;
305
+ };
306
+ coveragePolicy?: CoveragePolicyWire;
307
+ requiredClock?: RequiredClockWire;
308
+ initialSelection?: ArchiveSelectionWire;
309
+ expectedCanonicalSchema?: BackfillRequestWire["expected_canonical_schema"];
310
+ productPins?: BackfillRequestWire["product_pins"];
311
+ productionAuthorizationId?: string;
312
+ wire?: BackfillRequestWire;
313
+ };
314
+ export declare function decodeBackfillRunDocuments(input: {
315
+ request: unknown;
316
+ requiredClock: unknown;
317
+ }): MarketDataVendorBackfillRequest;
318
+ type IdempotencyBusinessFields = Omit<MarketDataVendorBackfillRequest, "requestId" | "idempotencyKey">;
319
+ export declare function backfillBusinessFields(request: Omit<MarketDataVendorBackfillRequest, "idempotencyKey"> & {
320
+ idempotencyKey?: string;
321
+ }): IdempotencyBusinessFields;
322
+ export declare function createBackfillIdempotencyKey(request: Omit<MarketDataVendorBackfillRequest, "idempotencyKey"> & {
323
+ idempotencyKey?: string;
324
+ }): string;
325
+ export declare function createBackfillIdempotencyKey(request: Omit<BackfillRequestWire, "idempotency_key"> & {
326
+ idempotency_key?: string;
327
+ }): string;
328
+ export declare function createBackfillIdempotencyKey(request: Record<string, unknown>): string;
329
+ export declare function parseBackfillRequest(value: unknown): MarketDataVendorBackfillRequest;
330
+ export declare const BACKFILL_STATUSES: readonly ["already_covered", "promoted", "capability_unsupported", "credentials_missing", "vendor_fetch_failed", "archive_ingest_failed", "promotion_verification_failed", "post_backfill_coverage_insufficient"];
331
+ export type BackfillStatus = (typeof BACKFILL_STATUSES)[number];
332
+ export type ProviderCapability = {
333
+ provider: "cryptohftdata";
334
+ adapterVersion: string;
335
+ providerExchangeId: string;
336
+ resolvedSymbol: string;
337
+ };
338
+ export type ProviderObjectEvidence = {
339
+ identity: string;
340
+ checksum: string;
341
+ bytes: number;
342
+ rows: number;
343
+ };
344
+ export type BackfillArchiveRow = {
345
+ table: "market_data.cex_order_book_levels" | "market_data.cex_order_book_depth_summary" | "market_data.cex_order_book_capture_promotions" | "market_data.cex_order_book_capture_qualifications" | "market_data.cex_order_book_archive_selections";
346
+ row: Record<string, unknown>;
347
+ };
348
+ export type ForwarderBatch = {
349
+ source: typeof EXTERNAL_BACKFILL_SOURCE;
350
+ deployment_id: string;
351
+ batch_id: string;
352
+ rows: BackfillArchiveRow[];
353
+ };
354
+ export type PromotionReceipt = {
355
+ schemaVersion: typeof BACKFILL_PROMOTION_SCHEMA_VERSION;
356
+ receiptId: string;
357
+ requestId: string;
358
+ idempotencyKey: string;
359
+ status: "passing";
360
+ source: typeof EXTERNAL_BACKFILL_SOURCE;
361
+ provider: "cryptohftdata";
362
+ adapterVersion: string;
363
+ captureBundleId: string;
364
+ exchange: string;
365
+ tradingPair: string;
366
+ marketType: "spot" | "swap" | "future";
367
+ feed: "ORDERBOOK";
368
+ startTimeMs: number;
369
+ endTimeMs: number;
370
+ depth: number;
371
+ constructionMode: "sampled_top_n_snapshot" | "exact_l2_reconstruction";
372
+ canonicalSchemaVersion: string;
373
+ checksumAlgorithm: "sha256-canonical-json-v1";
374
+ vendorSemanticDigest: string;
375
+ canonicalSemanticDigest: string;
376
+ prefixDigest: string;
377
+ suffixDigest: string;
378
+ seamVerified: true;
379
+ coverageVerified: true;
380
+ datasetObjects: ProviderObjectEvidence[];
381
+ verificationTimeMs: number;
382
+ };
383
+ export declare const promotionReceiptSchema: z.ZodType<PromotionReceipt>;
384
+ export type BackfillResult = {
385
+ schemaVersion: typeof BACKFILL_RESULT_SCHEMA_VERSION;
386
+ requestId: string;
387
+ idempotencyKey?: string;
388
+ status: BackfillStatus;
389
+ reasonCode: string;
390
+ receipt?: PromotionReceipt;
391
+ diagnostics?: Record<string, string | number | boolean>;
392
+ };
393
+ export {};
@@ -0,0 +1,100 @@
1
+ import { type ArchiveSelectionWire, type BackfillArchiveRow, type FinalBackfillStatus, type ForwarderBatch, type MarketDataVendorBackfillRequest, type PromotionReceiptWire, type ProviderCapability, type ProviderObjectEvidence } from "./contracts";
2
+ export type ProviderDataset<Row = unknown> = {
3
+ objects: ProviderObjectEvidence[];
4
+ vendorSemanticDigest: string;
5
+ rows: Row[];
6
+ };
7
+ export type NormalizedBackfill = {
8
+ captureBundleId: string;
9
+ objects: ProviderObjectEvidence[];
10
+ rows: BackfillArchiveRow[];
11
+ vendorSemanticDigest: string;
12
+ canonicalSemanticDigest: string;
13
+ };
14
+ export type CandidateVerification = {
15
+ passed: boolean;
16
+ captureBundleId: string;
17
+ canonicalSemanticDigest: string;
18
+ prefixDigest: string;
19
+ suffixDigest: string;
20
+ seamVerified: boolean;
21
+ coverageVerified: boolean;
22
+ reasonCode?: string;
23
+ };
24
+ export type ArchiveClusterIdentity = {
25
+ environment: string;
26
+ cluster: string;
27
+ };
28
+ export type ProductionForwarderAuthorization = {
29
+ authorizationId: string;
30
+ scope: "production";
31
+ environment: string;
32
+ cluster: string;
33
+ expiresAt: string;
34
+ credentialValidated: true;
35
+ };
36
+ export type ArchivePreflightResolution = {
37
+ selection: ArchiveSelectionWire;
38
+ receipts: PromotionReceiptWire[];
39
+ readerIdentity: ArchiveClusterIdentity;
40
+ verificationBaseline?: {
41
+ prefixDigest: string;
42
+ suffixDigest: string;
43
+ };
44
+ };
45
+ export type ForwarderPreflightResolution = {
46
+ forwarderIdentity: ArchiveClusterIdentity;
47
+ authorization: ProductionForwarderAuthorization;
48
+ };
49
+ /** @deprecated Use ArchivePreflightResolution and its exact selection. */
50
+ export type QualifiedCoverage = {
51
+ complete: boolean;
52
+ coverageDigest: string;
53
+ prefixDigest?: string;
54
+ suffixDigest?: string;
55
+ };
56
+ export type BackfillDomainOutcome = {
57
+ status: FinalBackfillStatus;
58
+ reasonCode: string;
59
+ reasonSubcode?: string;
60
+ requestId?: string;
61
+ idempotencyKey?: string;
62
+ target?: ArchiveClusterIdentity;
63
+ selection?: ArchiveSelectionWire;
64
+ receipt?: PromotionReceiptWire;
65
+ diagnostics?: Record<string, string | number | boolean>;
66
+ };
67
+ export type BackfillDependencies = {
68
+ archive: {
69
+ resolveSelection(request: MarketDataVendorBackfillRequest): Promise<ArchivePreflightResolution>;
70
+ verifyCandidate(request: MarketDataVendorBackfillRequest, normalized: NormalizedBackfill, captureBundleId: string, baseline: ArchivePreflightResolution): Promise<CandidateVerification>;
71
+ };
72
+ providers: {
73
+ capabilityFor(request: MarketDataVendorBackfillRequest): ProviderCapability | undefined;
74
+ acquire(request: MarketDataVendorBackfillRequest, capability: ProviderCapability, credential: unknown): Promise<ProviderDataset>;
75
+ normalize(request: MarketDataVendorBackfillRequest, capability: ProviderCapability, dataset: ProviderDataset, captureBundleId: string): Promise<NormalizedBackfill>;
76
+ };
77
+ credentials: {
78
+ resolve(provider: "cryptohftdata"): Promise<unknown | undefined>;
79
+ };
80
+ forwarder: {
81
+ preflight(input: {
82
+ authorizationId: string;
83
+ target: ArchiveClusterIdentity;
84
+ }): Promise<ForwarderPreflightResolution>;
85
+ submit(batch: ForwarderBatch): Promise<{
86
+ ok: boolean;
87
+ inserted: number;
88
+ }>;
89
+ };
90
+ clock: {
91
+ nowMs(): number;
92
+ };
93
+ retry?: {
94
+ maxAttempts: number;
95
+ wait?: (completedAttempt: number) => Promise<void>;
96
+ };
97
+ log?: (event: string, fields: Record<string, unknown>) => void;
98
+ };
99
+ export declare function createMarketDataVendorBackfillDependencies(dependencies: BackfillDependencies): BackfillDependencies;
100
+ export declare function runMarketDataVendorBackfill(input: unknown, dependencies: BackfillDependencies): Promise<BackfillDomainOutcome>;