@usherlabs/cex-broker 0.2.49 → 0.2.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/dist/commands/cex-canonical-orderbook-export.js +7747 -883
  2. package/dist/commands/cex-canonical-orderbook-export.js.map +41 -10
  3. package/dist/commands/cli.js +127 -34
  4. package/dist/commands/market-data-vendor-backfill.js +8284 -6391
  5. package/dist/commands/market-data-vendor-backfill.js.map +16 -15
  6. package/dist/helpers/canonical-orderbook-export/parquet-projection.d.ts +19 -0
  7. package/dist/helpers/market-data-archive/canonical-orderbook.d.ts +1 -1
  8. package/dist/helpers/market-data-preparation/conformance-fixtures.d.ts +114 -2
  9. package/dist/helpers/market-data-preparation/contracts.d.ts +1298 -18
  10. package/dist/helpers/market-data-preparation/required-clock-qualification.d.ts +43 -0
  11. package/dist/helpers/market-data-preparation/source-tape-dependencies.d.ts +19 -0
  12. package/dist/helpers/market-data-preparation/source-tape-operation.d.ts +109 -0
  13. package/dist/helpers/market-data-source-forensics.d.ts +400 -0
  14. package/dist/helpers/market-data-vendor-backfill/conformance-fixtures.d.ts +3 -5
  15. package/dist/helpers/market-data-vendor-backfill/contracts.d.ts +21 -56
  16. package/dist/helpers/market-data-vendor-backfill/core.d.ts +1 -0
  17. package/dist/helpers/market-data-vendor-backfill/cryptohftdata.d.ts +45 -5
  18. package/dist/helpers/market-data-vendor-backfill/manifests.d.ts +31 -1083
  19. package/dist/helpers/market-data-vendor-backfill/promotion.d.ts +1 -0
  20. package/dist/helpers/source-tape-sandbox.d.ts +82 -0
  21. package/dist/helpers/source-tape.d.ts +165 -0
  22. package/dist/helpers/trace-context.d.ts +8 -0
  23. package/dist/index.js +128 -35
  24. package/dist/index.js.map +7 -6
  25. package/dist/market-data-preparation/fixtures/{conformance-v2.json → conformance-v3.json} +257 -60
  26. package/dist/market-data-preparation/policies/capability-policy.json +15 -4
  27. package/dist/market-data-preparation/policies/resource-policy.json +3 -3
  28. package/dist/market-data-preparation/policies/source-tape-capability-v1.json +35 -0
  29. package/dist/market-data-preparation/schema-manifest.json +30 -10
  30. package/dist/market-data-preparation/schemas/canonical-orderbook-export-request-v1.schema.json +5 -1
  31. package/dist/market-data-preparation/schemas/{canonical-orderbook-export-result-v1.schema.json → canonical-orderbook-export-result-v2.schema.json} +20 -6
  32. package/dist/market-data-preparation/schemas/order-book-depth-summary-parquet-projection-v1.schema.json +256 -0
  33. package/dist/market-data-preparation/schemas/order-book-levels-parquet-projection-v1.schema.json +220 -0
  34. package/dist/market-data-preparation/schemas/{preparation-product-pin-v1.schema.json → preparation-product-pin-v2.schema.json} +59 -9
  35. package/dist/market-data-preparation/schemas/promotion-receipt-v1.schema.json +5 -1
  36. package/dist/market-data-preparation/schemas/source-forensics-ledger-v1.schema.json +463 -0
  37. package/dist/market-data-preparation/schemas/source-qualification-record-v1.schema.json +394 -0
  38. package/dist/market-data-preparation.d.ts +7 -0
  39. package/dist/market-data-preparation.js +35443 -25203
  40. package/dist/market-data-preparation.js.map +118 -78
  41. package/dist/market-data-vendor-backfill.d.ts +8 -5
  42. package/dist/market-data-vendor-backfill.js +19444 -15553
  43. package/dist/market-data-vendor-backfill.js.map +65 -60
  44. package/package.json +1 -5
  45. package/dist/market-data-preparation/policies/capability-policy-v1.json +0 -28
  46. package/dist/market-data-vendor-backfill/fixtures/conformance-v1.json +0 -448
  47. package/dist/market-data-vendor-backfill/policies/capability-policy-v2.json +0 -28
  48. package/dist/market-data-vendor-backfill/policies/capability-policy.json +0 -28
  49. package/dist/market-data-vendor-backfill/policies/resource-policy.json +0 -16
  50. package/dist/market-data-vendor-backfill/schema-manifest.json +0 -31
  51. package/dist/market-data-vendor-backfill/schemas/archive-selection.schema.json +0 -224
  52. package/dist/market-data-vendor-backfill/schemas/promotion-receipt.schema.json +0 -171
  53. package/dist/market-data-vendor-backfill/schemas/request.schema.json +0 -166
  54. package/dist/market-data-vendor-backfill/schemas/required-clock.schema.json +0 -47
  55. package/dist/market-data-vendor-backfill/schemas/result.schema.json +0 -198
@@ -1,18 +1,17 @@
1
1
  {
2
- "fixture_id": "cex-market-data-preparation-conformance/v2",
3
- "legacy_fixture_id": "market-data-vendor-backfill-conformance/v1",
2
+ "fixture_id": "cex-market-data-preparation-conformance/v3",
4
3
  "documents": {
5
4
  "backfill_result": {
6
5
  "schema_id": "https://schemas.usher.so/market-data-vendor-backfill-result/v2",
7
6
  "job_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f120",
8
7
  "request_file_sha256": "7777777777777777777777777777777777777777777777777777777777777777",
9
- "schema_manifest_sha256": "0e226165c78c6217a765c09be15b6305a574e69b6e05e8607f399b468ba04d5b",
8
+ "schema_manifest_sha256": "00fda4a5f67dff1e142af7cf7e37f641a3a960acd6e995530aaf99f8b2f5c7bb",
10
9
  "producer": {
11
10
  "product_id": "market-data-vendor-backfill",
12
11
  "product_version": "market-data-vendor-backfill/v1",
13
12
  "package": {
14
13
  "name": "@usherlabs/cex-broker",
15
- "version": "0.2.47",
14
+ "version": "0.2.50",
16
15
  "git_head": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
17
16
  },
18
17
  "executable_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
@@ -22,18 +21,18 @@
22
21
  }
23
22
  },
24
23
  "capability_policy": {
25
- "policy_id": "market-data-vendor-backfill-capabilities/v1",
26
- "policy_sha256": "51f522286ad24ec55525c19cad5c2c37043f9325a5a8f20d838adb7fd71191bf"
24
+ "policy_id": "market-data-vendor-backfill-capabilities/v3",
25
+ "policy_sha256": "5fc751f9823c118a9f168bb6c258cd6dfb57c2a45cdbdbf5efe8e75f54426f78"
27
26
  },
28
27
  "resource_policy": {
29
- "policy_id": "market-data-vendor-backfill-resources/v1",
30
- "policy_sha256": "38d4f47cbd15794efed570801556a95f724561a8b45567d518c044da9d454142"
28
+ "policy_id": "market-data-vendor-backfill-resources/v2",
29
+ "policy_sha256": "5c095c9af4450a0dd110cae04215b9188c6522ab16f4524a41b6122b154160a2"
31
30
  },
32
31
  "started_at": "2026-08-20T12:00:01.000Z",
33
32
  "completed_at": "2026-08-20T12:00:03.000Z",
34
33
  "outcome": {
35
- "status": "promoted",
36
- "reason_code": "promotion_qualified",
34
+ "status": "already_covered",
35
+ "reason_code": "qualified_coverage_complete",
37
36
  "reason_subcode": null,
38
37
  "request_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f100",
39
38
  "idempotency_key": "2108b6e712cd118bbd1acaff2cd7f481e230b6818048a3973b0a5bcbe3305c95",
@@ -90,8 +89,8 @@
90
89
  "qualification": {
91
90
  "qualification_event_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f104",
92
91
  "state": "qualified",
93
- "receipt_id": "69705f79f1a12ef550949b6aceb8b4a1c49230a87022b080e77cb91145525663",
94
- "promotion_identity_sha256": "e1993e99a7aefae1a2c56248d2c6ff35c5755f8840d474aaa792abe4ab2c22b5"
92
+ "receipt_id": "d0f810b0695f318a4167be6b221a932e06f42c5bdb3b62ab79d7811a3892fa62",
93
+ "promotion_identity_sha256": "2a33d6c70f02056d41034634b6da91dbea60fa5cd5fca7b85177ecf338001d25"
95
94
  }
96
95
  }
97
96
  ],
@@ -105,18 +104,18 @@
105
104
  "metadata_ref": {
106
105
  "capture_origin": "vendor_historical_backfill",
107
106
  "qualification_event_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f104",
108
- "receipt_id": "69705f79f1a12ef550949b6aceb8b4a1c49230a87022b080e77cb91145525663"
107
+ "receipt_id": "d0f810b0695f318a4167be6b221a932e06f42c5bdb3b62ab79d7811a3892fa62"
109
108
  }
110
109
  }
111
110
  ],
112
111
  "receipt_ids": [
113
- "69705f79f1a12ef550949b6aceb8b4a1c49230a87022b080e77cb91145525663"
112
+ "d0f810b0695f318a4167be6b221a932e06f42c5bdb3b62ab79d7811a3892fa62"
114
113
  ],
115
114
  "qualification_event_ids": [
116
115
  "018f0f4d-7b32-7a30-8f4d-1d2a6e40f104"
117
116
  ],
118
117
  "resolved_at": "2026-08-20T12:00:02.000Z",
119
- "selection_sha256": "cc4869df6d08c79d030c8407e7a55e5c534e6a96da7c1ac3f726e814c66a108e"
118
+ "selection_sha256": "9506a245eb2c0825490a23b630b649c0b8f2d681bf7c047865bd356e1d69dcd3"
120
119
  },
121
120
  "receipt": {
122
121
  "schema_id": "https://schemas.usher.so/market-data-vendor-backfill-promotion-receipt/v1",
@@ -130,20 +129,20 @@
130
129
  "adapter_version": "cryptohftdata-orderbook/v2",
131
130
  "effective_policies": {
132
131
  "capability_policy": {
133
- "policy_id": "market-data-vendor-backfill-capabilities/v1",
134
- "policy_sha256": "51f522286ad24ec55525c19cad5c2c37043f9325a5a8f20d838adb7fd71191bf"
132
+ "policy_id": "market-data-vendor-backfill-capabilities/v3",
133
+ "policy_sha256": "5fc751f9823c118a9f168bb6c258cd6dfb57c2a45cdbdbf5efe8e75f54426f78"
135
134
  },
136
135
  "resource_policy": {
137
- "policy_id": "market-data-vendor-backfill-resources/v1",
138
- "policy_sha256": "38d4f47cbd15794efed570801556a95f724561a8b45567d518c044da9d454142"
136
+ "policy_id": "market-data-vendor-backfill-resources/v2",
137
+ "policy_sha256": "5c095c9af4450a0dd110cae04215b9188c6522ab16f4524a41b6122b154160a2"
139
138
  },
140
139
  "adapter_policy": {
141
140
  "policy_id": "cryptohftdata-orderbook-adapter/v1",
142
141
  "policy_sha256": "b71eec2948593eb3f9f43371a23ec6a37fd186177c6e368f7f792113e907a737"
143
142
  },
144
143
  "acquisition_policy": {
145
- "policy_id": "cryptohftdata-hourly-acquisition/v1",
146
- "policy_sha256": "dd3b94d6fdeed161580735673913e3483dada4725ee4fb7add461b74e20713d5"
144
+ "policy_id": "cryptohftdata-hourly-acquisition/v2",
145
+ "policy_sha256": "bc70353efd09ec49db97320f831da358191429c1eac42d053cce3ff4e61e320c"
147
146
  }
148
147
  },
149
148
  "capture_bundle_id": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
@@ -184,14 +183,12 @@
184
183
  "rows": 50
185
184
  }
186
185
  ],
187
- "promotion_identity_sha256": "e1993e99a7aefae1a2c56248d2c6ff35c5755f8840d474aaa792abe4ab2c22b5",
188
- "receipt_id": "69705f79f1a12ef550949b6aceb8b4a1c49230a87022b080e77cb91145525663"
186
+ "promotion_identity_sha256": "2a33d6c70f02056d41034634b6da91dbea60fa5cd5fca7b85177ecf338001d25",
187
+ "receipt_id": "d0f810b0695f318a4167be6b221a932e06f42c5bdb3b62ab79d7811a3892fa62"
189
188
  },
190
- "diagnostics": {
191
- "promoted_rows": 50
192
- }
189
+ "diagnostics": {}
193
190
  },
194
- "result_sha256": "a436b2f916fb2b2885a2694016227385a01e2d4cea8eda01417a8c8992d3317a"
191
+ "result_sha256": "22a07a36ed103b70676d3fa57f2ff944e25e994138fe48aee8092eadb344ab03"
195
192
  },
196
193
  "canonical_orderbook_export_request": {
197
194
  "schema_id": "https://schemas.usher.so/cex-canonical-orderbook-export-request/v1",
@@ -249,8 +246,8 @@
249
246
  "qualification": {
250
247
  "qualification_event_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f104",
251
248
  "state": "qualified",
252
- "receipt_id": "69705f79f1a12ef550949b6aceb8b4a1c49230a87022b080e77cb91145525663",
253
- "promotion_identity_sha256": "e1993e99a7aefae1a2c56248d2c6ff35c5755f8840d474aaa792abe4ab2c22b5"
249
+ "receipt_id": "d0f810b0695f318a4167be6b221a932e06f42c5bdb3b62ab79d7811a3892fa62",
250
+ "promotion_identity_sha256": "2a33d6c70f02056d41034634b6da91dbea60fa5cd5fca7b85177ecf338001d25"
254
251
  }
255
252
  }
256
253
  ],
@@ -264,18 +261,18 @@
264
261
  "metadata_ref": {
265
262
  "capture_origin": "vendor_historical_backfill",
266
263
  "qualification_event_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f104",
267
- "receipt_id": "69705f79f1a12ef550949b6aceb8b4a1c49230a87022b080e77cb91145525663"
264
+ "receipt_id": "d0f810b0695f318a4167be6b221a932e06f42c5bdb3b62ab79d7811a3892fa62"
268
265
  }
269
266
  }
270
267
  ],
271
268
  "receipt_ids": [
272
- "69705f79f1a12ef550949b6aceb8b4a1c49230a87022b080e77cb91145525663"
269
+ "d0f810b0695f318a4167be6b221a932e06f42c5bdb3b62ab79d7811a3892fa62"
273
270
  ],
274
271
  "qualification_event_ids": [
275
272
  "018f0f4d-7b32-7a30-8f4d-1d2a6e40f104"
276
273
  ],
277
274
  "resolved_at": "2026-08-20T12:00:02.000Z",
278
- "selection_sha256": "cc4869df6d08c79d030c8407e7a55e5c534e6a96da7c1ac3f726e814c66a108e"
275
+ "selection_sha256": "9506a245eb2c0825490a23b630b649c0b8f2d681bf7c047865bd356e1d69dcd3"
279
276
  },
280
277
  "depth": 20,
281
278
  "construction_mode": "sampled_top_n_snapshot",
@@ -283,15 +280,15 @@
283
280
  "checksum_algorithm": "sha256-canonical-json-v1"
284
281
  },
285
282
  "canonical_orderbook_export_result": {
286
- "schema_id": "https://schemas.usher.so/cex-canonical-orderbook-export-result/v1",
283
+ "schema_id": "https://schemas.usher.so/cex-canonical-orderbook-export-result/v2",
287
284
  "job_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f122",
288
285
  "request_file_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
289
286
  "producer": {
290
287
  "product_id": "cex-canonical-orderbook-export",
291
- "product_version": "cex-canonical-orderbook-export/v1",
288
+ "product_version": "cex-canonical-orderbook-export/v2",
292
289
  "package": {
293
290
  "name": "@usherlabs/cex-broker",
294
- "version": "0.2.47",
291
+ "version": "0.2.50",
295
292
  "git_head": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
296
293
  },
297
294
  "executable_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
@@ -311,7 +308,7 @@
311
308
  "environment": "production",
312
309
  "cluster": "cex-archive-primary"
313
310
  },
314
- "selection_sha256": "cc4869df6d08c79d030c8407e7a55e5c534e6a96da7c1ac3f726e814c66a108e",
311
+ "selection_sha256": "9506a245eb2c0825490a23b630b649c0b8f2d681bf7c047865bd356e1d69dcd3",
315
312
  "query_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
316
313
  "query_segments": [
317
314
  {
@@ -322,32 +319,36 @@
322
319
  }
323
320
  ],
324
321
  "promotion_receipt_ids": [
325
- "69705f79f1a12ef550949b6aceb8b4a1c49230a87022b080e77cb91145525663"
322
+ "d0f810b0695f318a4167be6b221a932e06f42c5bdb3b62ab79d7811a3892fa62"
326
323
  ],
327
324
  "artifacts": {
328
325
  "levels": {
329
326
  "file_name": "order_book_levels.parquet",
330
327
  "rows": 40,
331
328
  "bytes": 4096,
332
- "sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
329
+ "sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
330
+ "projection_schema_id": "https://schemas.usher.so/cex-order-book-levels-parquet-projection/v1",
331
+ "projection_schema_sha256": "dbf98aabe22f715388d7fa8d791935a3131fe8d372cfe7041296fd3ace501ffc"
333
332
  },
334
333
  "summary": {
335
334
  "file_name": "order_book_depth_summary.parquet",
336
335
  "rows": 1,
337
336
  "bytes": 2048,
338
- "sha256": "1111111111111111111111111111111111111111111111111111111111111111"
337
+ "sha256": "1111111111111111111111111111111111111111111111111111111111111111",
338
+ "projection_schema_id": "https://schemas.usher.so/cex-order-book-depth-summary-parquet-projection/v1",
339
+ "projection_schema_sha256": "7fca0794401e0a8de767c4f70bd3b258e20eee94565060c0749ebcb75588e9d1"
339
340
  }
340
341
  },
341
342
  "diagnostics": {}
342
343
  },
343
- "result_sha256": "4be9b182d1858bf4dd60da823e245f9a6ca472e99b3b50b94c17a48c71c9dc67"
344
+ "result_sha256": "4037956a82b3727acc6e373041de30f937c874c98d90cbd5e8533c11757d33d4"
344
345
  },
345
346
  "preparation_product_pin": {
346
- "schema_id": "cex-market-data-preparation-product-pin/v1",
347
+ "schema_id": "cex-market-data-preparation-product-pin/v2",
347
348
  "package": {
348
349
  "name": "@usherlabs/cex-broker",
349
- "version": "0.2.47",
350
- "registry_tarball_url": "https://registry.npmjs.org/@usherlabs/cex-broker/-/cex-broker-0.2.47.tgz",
350
+ "version": "0.2.50",
351
+ "registry_tarball_url": "https://registry.npmjs.org/@usherlabs/cex-broker/-/cex-broker-0.2.50.tgz",
351
352
  "integrity": "sha512-YQ==",
352
353
  "tarball_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
353
354
  "npm_git_head": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
@@ -361,14 +362,14 @@
361
362
  },
362
363
  {
363
364
  "product_id": "cex-canonical-orderbook-export",
364
- "product_version": "cex-canonical-orderbook-export/v1",
365
+ "product_version": "cex-canonical-orderbook-export/v2",
365
366
  "relative_path": "dist/commands/cex-canonical-orderbook-export.js",
366
367
  "executable_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
367
368
  }
368
369
  ],
369
370
  "schema_manifest": {
370
- "schema_id": "https://schemas.usher.so/market-data-vendor-backfill-schema-manifest/v2",
371
- "manifest_sha256": "0e226165c78c6217a765c09be15b6305a574e69b6e05e8607f399b468ba04d5b",
371
+ "schema_id": "https://schemas.usher.so/market-data-vendor-backfill-schema-manifest/v3",
372
+ "manifest_sha256": "00fda4a5f67dff1e142af7cf7e37f641a3a960acd6e995530aaf99f8b2f5c7bb",
372
373
  "relative_path": "dist/market-data-preparation/schema-manifest.json"
373
374
  },
374
375
  "schema_pins": [
@@ -390,35 +391,231 @@
390
391
  },
391
392
  {
392
393
  "schema_id": "https://schemas.usher.so/market-data-vendor-backfill-promotion-receipt/v1",
393
- "schema_sha256": "00c42aba3e4a26d4692ef5554273ed762c21fce8c07624bc970ff0b28884b099"
394
+ "schema_sha256": "95629b9d6d8544f84d42caac4e40c7995f4933836fbb974cb186d11293739438"
394
395
  },
395
396
  {
396
397
  "schema_id": "https://schemas.usher.so/cex-canonical-orderbook-export-request/v1",
397
- "schema_sha256": "50a698628b00204c927e28d8c7048345330fb17ab61f86addd3a04114debc0f3"
398
+ "schema_sha256": "f82096cd3e0fb8d513ad5a9dbdb817d1c6e3e56dbbde745d4d8ff3be7c1528eb"
399
+ },
400
+ {
401
+ "schema_id": "https://schemas.usher.so/cex-canonical-orderbook-export-result/v2",
402
+ "schema_sha256": "315dc2c6e00d5504c1152ddf00b1e8eabd18008293eb6aca3e696adc8b0e11d6"
403
+ },
404
+ {
405
+ "schema_id": "cex-market-data-preparation-product-pin/v2",
406
+ "schema_sha256": "03bb7b0ffc5293a6a2dbceb871ff75fc83e54a979cd22bc2c99e7751b761c833"
398
407
  },
399
408
  {
400
- "schema_id": "https://schemas.usher.so/cex-canonical-orderbook-export-result/v1",
401
- "schema_sha256": "77da86c140474294bfc351be6a3a69c57fff95032e9bafa9aaa09d398315cd3c"
409
+ "schema_id": "https://schemas.usher.so/cex-order-book-levels-parquet-projection/v1",
410
+ "schema_sha256": "dbf98aabe22f715388d7fa8d791935a3131fe8d372cfe7041296fd3ace501ffc"
402
411
  },
403
412
  {
404
- "schema_id": "cex-market-data-preparation-product-pin/v1",
405
- "schema_sha256": "7e50cf70aaa87fc608647dfc71e61f18c3da05f27f1f6ceba4c58bc960ff88ff"
413
+ "schema_id": "https://schemas.usher.so/cex-order-book-depth-summary-parquet-projection/v1",
414
+ "schema_sha256": "7fca0794401e0a8de767c4f70bd3b258e20eee94565060c0749ebcb75588e9d1"
415
+ },
416
+ {
417
+ "schema_id": "https://schemas.usher.so/market-data-source-forensics-ledger/v1",
418
+ "schema_sha256": "a0fbe689cb309acaadf8ae432890fc572b4db37763e8cdcf00c06e7307d08e97"
419
+ },
420
+ {
421
+ "schema_id": "https://schemas.usher.so/market-data-source-qualification-record/v1",
422
+ "schema_sha256": "2a9fa7c0b005bcfa79bc03714e0b7dc145cde7ebf071b8ed4ce48bb2ad53abd5"
406
423
  }
407
424
  ],
408
425
  "capability_policy": {
409
- "policy_id": "market-data-vendor-backfill-capabilities/v2",
410
- "policy_sha256": "a1bce5936b00ca6f8c8d338b57eddbaf6d48e6f9c40566c4beae76630e969c5a"
426
+ "policy_id": "market-data-vendor-backfill-capabilities/v3",
427
+ "policy_sha256": "5fc751f9823c118a9f168bb6c258cd6dfb57c2a45cdbdbf5efe8e75f54426f78"
428
+ },
429
+ "source_tape_capability": {
430
+ "policy_id": "market-data-source-tape-capability/v1",
431
+ "policy_sha256": "203300cc7060f018de073fa96eee7ba8210bbd5a768063166c1a7c2b2adcbe05"
411
432
  },
412
433
  "resource_policy": {
413
- "policy_id": "market-data-vendor-backfill-resources/v1",
414
- "policy_sha256": "38d4f47cbd15794efed570801556a95f724561a8b45567d518c044da9d454142"
434
+ "policy_id": "market-data-vendor-backfill-resources/v2",
435
+ "policy_sha256": "5c095c9af4450a0dd110cae04215b9188c6522ab16f4524a41b6122b154160a2"
436
+ },
437
+ "preparation_library": {
438
+ "exported_subpath": "@usherlabs/cex-broker/market-data-preparation",
439
+ "runtime_entry_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
440
+ "declaration_sha256": "4444444444444444444444444444444444444444444444444444444444444444",
441
+ "operations": [
442
+ {
443
+ "symbol": "runMarketDataSourceTape",
444
+ "operation_id": "market-data-source-tape/v1"
445
+ },
446
+ {
447
+ "symbol": "runMarketDataRequiredClockQualification",
448
+ "operation_id": "market-data-required-clock-qualification/v1"
449
+ }
450
+ ]
451
+ }
452
+ },
453
+ "library_operations": {
454
+ "required_clock_qualification": {
455
+ "operation_id": "market-data-required-clock-qualification/v1",
456
+ "attempt_id": "arb-usdt-required-clock-conformance",
457
+ "request": {
458
+ "schema_id": "https://schemas.usher.so/market-data-vendor-backfill-request/v1",
459
+ "request_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f100",
460
+ "attempt_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f101",
461
+ "scope": {
462
+ "exchange": "okx",
463
+ "trading_pair": "ARB-USDT",
464
+ "market_type": "spot",
465
+ "feed": "ORDERBOOK"
466
+ },
467
+ "window": {
468
+ "start_at": "2026-08-18T09:27:15.000Z",
469
+ "end_at": "2026-08-18T09:27:17.000Z"
470
+ },
471
+ "depth": 20,
472
+ "construction_mode": "sampled_top_n_snapshot",
473
+ "source_policy": "authoritative_window",
474
+ "target": {
475
+ "environment": "production",
476
+ "cluster": "cex-archive-primary"
477
+ },
478
+ "coverage_policy": {
479
+ "policy_id": "prior-asof-strict/v1",
480
+ "max_asof_lag_ms": 5000,
481
+ "future_rows": "reject",
482
+ "missing_required_event": "fail"
483
+ },
484
+ "required_clock": {
485
+ "schema_id": "https://schemas.usher.so/market-data-vendor-backfill-required-clock/v1",
486
+ "clock_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f102",
487
+ "file_name": "required-clock.json",
488
+ "clock_sha256": "1934fd40540efffee5f5e862cea676f65e398c129db9a3f988acd28765ff0600",
489
+ "event_count": 1
490
+ },
491
+ "initial_selection": {
492
+ "schema_id": "https://schemas.usher.so/market-data-vendor-backfill-archive-selection/v1",
493
+ "scope": {
494
+ "exchange": "okx",
495
+ "trading_pair": "ARB-USDT",
496
+ "market_type": "spot",
497
+ "feed": "ORDERBOOK"
498
+ },
499
+ "required_clock": {
500
+ "clock_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f102",
501
+ "clock_sha256": "1934fd40540efffee5f5e862cea676f65e398c129db9a3f988acd28765ff0600",
502
+ "event_count": 1
503
+ },
504
+ "coverage_policy": {
505
+ "policy_id": "prior-asof-strict/v1",
506
+ "max_asof_lag_ms": 5000,
507
+ "future_rows": "reject",
508
+ "missing_required_event": "fail"
509
+ },
510
+ "source_policy": "authoritative_window",
511
+ "coverage_class": "missing",
512
+ "requested_intervals": [
513
+ {
514
+ "start_at": "2026-08-18T09:27:15.000Z",
515
+ "end_at": "2026-08-18T09:27:17.000Z"
516
+ }
517
+ ],
518
+ "selected_intervals": [],
519
+ "precedence": [
520
+ "vendor"
521
+ ],
522
+ "bundles": [],
523
+ "support_anchors": [],
524
+ "receipt_ids": [],
525
+ "qualification_event_ids": [],
526
+ "resolved_at": "2026-08-20T12:00:00.000Z",
527
+ "selection_sha256": "7f9a03dbf3dd3c600ffe7a28bd2b2211a69509c26c06d5e11ad926faeb82f895"
528
+ },
529
+ "expected_canonical_schema": {
530
+ "schema_id": "cex-order-book-canonical/v1",
531
+ "schema_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
532
+ },
533
+ "product_pins": {
534
+ "capability_policy": {
535
+ "policy_id": "market-data-vendor-backfill-capabilities/v3",
536
+ "policy_sha256": "5fc751f9823c118a9f168bb6c258cd6dfb57c2a45cdbdbf5efe8e75f54426f78"
537
+ },
538
+ "resource_policy": {
539
+ "policy_id": "market-data-vendor-backfill-resources/v2",
540
+ "policy_sha256": "5c095c9af4450a0dd110cae04215b9188c6522ab16f4524a41b6122b154160a2"
541
+ }
542
+ },
543
+ "production_authorization_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f103",
544
+ "idempotency_key": "2108b6e712cd118bbd1acaff2cd7f481e230b6818048a3973b0a5bcbe3305c95"
545
+ },
546
+ "required_clock": {
547
+ "schema_id": "https://schemas.usher.so/market-data-vendor-backfill-required-clock/v1",
548
+ "clock_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f102",
549
+ "created_at": "2026-08-20T12:00:00.000Z",
550
+ "targets": [
551
+ {
552
+ "target_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f105",
553
+ "target_at": "2026-08-18T09:27:16.000Z"
554
+ }
555
+ ],
556
+ "clock_sha256": "1934fd40540efffee5f5e862cea676f65e398c129db9a3f988acd28765ff0600"
557
+ },
558
+ "artifacts": {
559
+ "ledger_file_name": "required-clock-ledger.json",
560
+ "qualification_record_file_name": "required-clock-qualification.json"
561
+ }
562
+ },
563
+ "source_tape": {
564
+ "operation_id": "market-data-source-tape/v1",
565
+ "attempt_id": "arb-usdt-source-tape-conformance",
566
+ "request_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f100",
567
+ "scope": {
568
+ "exchange": "okx",
569
+ "trading_pair": "ARB-USDT",
570
+ "market_type": "spot",
571
+ "feed": "ORDERBOOK"
572
+ },
573
+ "window": {
574
+ "start_at": "2026-08-18T09:27:15.000Z",
575
+ "end_at": "2026-08-18T09:27:17.000Z"
576
+ },
577
+ "depth": 100,
578
+ "target": {
579
+ "environment": "sandbox",
580
+ "cluster": "cex-archive-local"
581
+ },
582
+ "production_authorization_id": "018f0f4d-7b32-7a30-8f4d-1d2a6e40f103",
583
+ "expected_canonical_schema": {
584
+ "schema_id": "cex-order-book-canonical/v1",
585
+ "schema_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
586
+ },
587
+ "product_pins": {
588
+ "source_tape_capability": {
589
+ "policy_id": "market-data-source-tape-capability/v1",
590
+ "policy_sha256": "203300cc7060f018de073fa96eee7ba8210bbd5a768063166c1a7c2b2adcbe05"
591
+ },
592
+ "resource_policy": {
593
+ "policy_id": "market-data-vendor-backfill-resources/v2",
594
+ "policy_sha256": "5c095c9af4450a0dd110cae04215b9188c6522ab16f4524a41b6122b154160a2"
595
+ },
596
+ "adapter_policy": {
597
+ "policy_id": "cryptohftdata-orderbook-adapter/v1",
598
+ "policy_sha256": "b71eec2948593eb3f9f43371a23ec6a37fd186177c6e368f7f792113e907a737"
599
+ },
600
+ "acquisition_policy": {
601
+ "policy_id": "cryptohftdata-hourly-acquisition/v2",
602
+ "policy_sha256": "bc70353efd09ec49db97320f831da358191429c1eac42d053cce3ff4e61e320c"
603
+ }
604
+ },
605
+ "artifacts": {
606
+ "ledger_file_name": "source-tape-ledger.json",
607
+ "qualification_record_file_name": "source-tape-qualification.json",
608
+ "exporter_result_file_name": "source-tape-export-result.json"
609
+ }
415
610
  }
416
611
  }
417
612
  },
418
613
  "identities": {
419
- "manifest_sha256": "0e226165c78c6217a765c09be15b6305a574e69b6e05e8607f399b468ba04d5b",
420
- "backfill_result_sha256": "a436b2f916fb2b2885a2694016227385a01e2d4cea8eda01417a8c8992d3317a",
421
- "selection_sha256": "cc4869df6d08c79d030c8407e7a55e5c534e6a96da7c1ac3f726e814c66a108e",
422
- "export_result_sha256": "4be9b182d1858bf4dd60da823e245f9a6ca472e99b3b50b94c17a48c71c9dc67"
614
+ "manifest_sha256": "00fda4a5f67dff1e142af7cf7e37f641a3a960acd6e995530aaf99f8b2f5c7bb",
615
+ "backfill_result_sha256": "22a07a36ed103b70676d3fa57f2ff944e25e994138fe48aee8092eadb344ab03",
616
+ "selection_sha256": "9506a245eb2c0825490a23b630b649c0b8f2d681bf7c047865bd356e1d69dcd3",
617
+ "export_result_sha256": "4037956a82b3727acc6e373041de30f937c874c98d90cbd5e8533c11757d33d4",
618
+ "product_pin_sha256": "888afea6a5748146882f2b2fad87838a4b854aad9b047296064e84f8eb0418a2",
619
+ "library_operations_sha256": "2dba0231b99379a317db35e6200e0469991bc97cacce0da3a245014b491167d5"
423
620
  }
424
621
  }
@@ -1,15 +1,15 @@
1
1
  {
2
- "policy_id": "market-data-vendor-backfill-capabilities/v2",
2
+ "policy_id": "market-data-vendor-backfill-capabilities/v3",
3
3
  "provider": "cryptohftdata",
4
4
  "adapter_policy": {
5
5
  "policy_id": "cryptohftdata-orderbook-adapter/v1",
6
6
  "adapter_version": "cryptohftdata-orderbook/v2"
7
7
  },
8
8
  "acquisition_policy": {
9
- "policy_id": "cryptohftdata-hourly-acquisition/v1",
9
+ "policy_id": "cryptohftdata-hourly-acquisition/v2",
10
10
  "object_granularity": "utc_hour",
11
11
  "authentication": "bearer",
12
- "initialization_lookback_ms": 0
12
+ "initialization_lookback_ms": 604800000
13
13
  },
14
14
  "profiles": [
15
15
  {
@@ -22,7 +22,18 @@
22
22
  "construction_modes": ["sampled_top_n_snapshot"],
23
23
  "source_policies": ["authoritative_window", "fill_gaps"],
24
24
  "max_depth": 400
25
+ },
26
+ {
27
+ "exchange": "okx",
28
+ "market_type": "spot",
29
+ "feed": "ORDERBOOK",
30
+ "canonical_trading_pair": "ARB-USDC",
31
+ "provider_exchange_id": "okx_spot",
32
+ "resolved_symbol": "ARB-USDC",
33
+ "construction_modes": ["sampled_top_n_snapshot"],
34
+ "source_policies": ["authoritative_window", "fill_gaps"],
35
+ "max_depth": 400
25
36
  }
26
37
  ],
27
- "policy_sha256": "a1bce5936b00ca6f8c8d338b57eddbaf6d48e6f9c40566c4beae76630e969c5a"
38
+ "policy_sha256": "5fc751f9823c118a9f168bb6c258cd6dfb57c2a45cdbdbf5efe8e75f54426f78"
28
39
  }
@@ -1,5 +1,5 @@
1
1
  {
2
- "policy_id": "market-data-vendor-backfill-resources/v1",
2
+ "policy_id": "market-data-vendor-backfill-resources/v2",
3
3
  "limits": {
4
4
  "max_files": 10000,
5
5
  "max_bytes": 107374182400,
@@ -9,8 +9,8 @@
9
9
  },
10
10
  "request_bounds": {
11
11
  "max_depth": 500,
12
- "max_window_ms": 604800000,
12
+ "max_window_ms": 2678400000,
13
13
  "max_required_events": 100000
14
14
  },
15
- "policy_sha256": "38d4f47cbd15794efed570801556a95f724561a8b45567d518c044da9d454142"
15
+ "policy_sha256": "5c095c9af4450a0dd110cae04215b9188c6522ab16f4524a41b6122b154160a2"
16
16
  }
@@ -0,0 +1,35 @@
1
+ {
2
+ "policy_id": "market-data-source-tape-capability/v1",
3
+ "provider": "cryptohftdata",
4
+ "adapter_version": "cryptohftdata-orderbook/v2",
5
+ "acquisition_policy_id": "cryptohftdata-hourly-acquisition/v2",
6
+ "exchange": "okx",
7
+ "market_type": "spot",
8
+ "feed": "ORDERBOOK",
9
+ "trading_pairs": ["ARB-USDC", "ARB-USDT"],
10
+ "construction_mode": "policy_neutral_top_n_state_change_tape/v1",
11
+ "canonical_schema_id": "cex-order-book-canonical/v1",
12
+ "depth": 100,
13
+ "source_window_scan": "complete_independent_of_required_clock",
14
+ "provider_object_inventory": "positive_expected_observed_equality",
15
+ "state_emission": "initialization_then_every_sequence_valid_state_or_freshness_change",
16
+ "state_window": "one_support_state_then_changes_in_half_open_window",
17
+ "archive_target": {
18
+ "environment": "sandbox",
19
+ "cluster": "cex-archive-local"
20
+ },
21
+ "archive_path": "normal_forwarder_promotion_qualification_selection_exporter_v2",
22
+ "submission_bounds": {
23
+ "max_states_per_yield": 4,
24
+ "max_rows_per_batch": 1000,
25
+ "max_bytes_per_batch": 5242880,
26
+ "max_in_flight": 1
27
+ },
28
+ "clickhouse_image": "clickhouse/clickhouse-server:24.8.14.39@sha256:1ffa82edee000a42c09313bd9f1293d94c570aee74babc1b3ca9983a35fa597b",
29
+ "clickhouse_server_version": "24.8.14.39",
30
+ "projection_schema_ids": [
31
+ "https://schemas.usher.so/cex-order-book-levels-parquet-projection/v1",
32
+ "https://schemas.usher.so/cex-order-book-depth-summary-parquet-projection/v1"
33
+ ],
34
+ "policy_sha256": "203300cc7060f018de073fa96eee7ba8210bbd5a768063166c1a7c2b2adcbe05"
35
+ }
@@ -1,5 +1,5 @@
1
1
  {
2
- "schema_id": "https://schemas.usher.so/market-data-vendor-backfill-schema-manifest/v2",
2
+ "schema_id": "https://schemas.usher.so/market-data-vendor-backfill-schema-manifest/v3",
3
3
  "artifacts": [
4
4
  {
5
5
  "schema_id": "https://schemas.usher.so/market-data-vendor-backfill-request/v1",
@@ -24,23 +24,43 @@
24
24
  {
25
25
  "schema_id": "https://schemas.usher.so/market-data-vendor-backfill-promotion-receipt/v1",
26
26
  "path": "schemas/promotion-receipt-v1.schema.json",
27
- "schema_sha256": "00c42aba3e4a26d4692ef5554273ed762c21fce8c07624bc970ff0b28884b099"
27
+ "schema_sha256": "95629b9d6d8544f84d42caac4e40c7995f4933836fbb974cb186d11293739438"
28
28
  },
29
29
  {
30
30
  "schema_id": "https://schemas.usher.so/cex-canonical-orderbook-export-request/v1",
31
31
  "path": "schemas/canonical-orderbook-export-request-v1.schema.json",
32
- "schema_sha256": "50a698628b00204c927e28d8c7048345330fb17ab61f86addd3a04114debc0f3"
32
+ "schema_sha256": "f82096cd3e0fb8d513ad5a9dbdb817d1c6e3e56dbbde745d4d8ff3be7c1528eb"
33
33
  },
34
34
  {
35
- "schema_id": "https://schemas.usher.so/cex-canonical-orderbook-export-result/v1",
36
- "path": "schemas/canonical-orderbook-export-result-v1.schema.json",
37
- "schema_sha256": "77da86c140474294bfc351be6a3a69c57fff95032e9bafa9aaa09d398315cd3c"
35
+ "schema_id": "https://schemas.usher.so/cex-canonical-orderbook-export-result/v2",
36
+ "path": "schemas/canonical-orderbook-export-result-v2.schema.json",
37
+ "schema_sha256": "315dc2c6e00d5504c1152ddf00b1e8eabd18008293eb6aca3e696adc8b0e11d6"
38
38
  },
39
39
  {
40
- "schema_id": "cex-market-data-preparation-product-pin/v1",
41
- "path": "schemas/preparation-product-pin-v1.schema.json",
42
- "schema_sha256": "7e50cf70aaa87fc608647dfc71e61f18c3da05f27f1f6ceba4c58bc960ff88ff"
40
+ "schema_id": "cex-market-data-preparation-product-pin/v2",
41
+ "path": "schemas/preparation-product-pin-v2.schema.json",
42
+ "schema_sha256": "03bb7b0ffc5293a6a2dbceb871ff75fc83e54a979cd22bc2c99e7751b761c833"
43
+ },
44
+ {
45
+ "schema_id": "https://schemas.usher.so/cex-order-book-levels-parquet-projection/v1",
46
+ "path": "schemas/order-book-levels-parquet-projection-v1.schema.json",
47
+ "schema_sha256": "dbf98aabe22f715388d7fa8d791935a3131fe8d372cfe7041296fd3ace501ffc"
48
+ },
49
+ {
50
+ "schema_id": "https://schemas.usher.so/cex-order-book-depth-summary-parquet-projection/v1",
51
+ "path": "schemas/order-book-depth-summary-parquet-projection-v1.schema.json",
52
+ "schema_sha256": "7fca0794401e0a8de767c4f70bd3b258e20eee94565060c0749ebcb75588e9d1"
53
+ },
54
+ {
55
+ "schema_id": "https://schemas.usher.so/market-data-source-forensics-ledger/v1",
56
+ "path": "schemas/source-forensics-ledger-v1.schema.json",
57
+ "schema_sha256": "a0fbe689cb309acaadf8ae432890fc572b4db37763e8cdcf00c06e7307d08e97"
58
+ },
59
+ {
60
+ "schema_id": "https://schemas.usher.so/market-data-source-qualification-record/v1",
61
+ "path": "schemas/source-qualification-record-v1.schema.json",
62
+ "schema_sha256": "2a9fa7c0b005bcfa79bc03714e0b7dc145cde7ebf071b8ed4ce48bb2ad53abd5"
43
63
  }
44
64
  ],
45
- "manifest_sha256": "0e226165c78c6217a765c09be15b6305a574e69b6e05e8607f399b468ba04d5b"
65
+ "manifest_sha256": "00fda4a5f67dff1e142af7cf7e37f641a3a960acd6e995530aaf99f8b2f5c7bb"
46
66
  }