@usherlabs/cex-broker 0.2.50 → 0.2.52

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/dist/commands/cex-canonical-orderbook-export.js +7750 -920
  2. package/dist/commands/cex-canonical-orderbook-export.js.map +41 -10
  3. package/dist/commands/cli.js +219 -24
  4. package/dist/commands/market-data-vendor-backfill.js +8297 -6460
  5. package/dist/commands/market-data-vendor-backfill.js.map +16 -15
  6. package/dist/helpers/canonical-orderbook-export/parquet-projection.d.ts +19 -0
  7. package/dist/helpers/deposit-archive-poller.d.ts +33 -0
  8. package/dist/helpers/market-data-archive/canonical-orderbook.d.ts +1 -1
  9. package/dist/helpers/market-data-preparation/conformance-fixtures.d.ts +114 -2
  10. package/dist/helpers/market-data-preparation/contracts.d.ts +1298 -18
  11. package/dist/helpers/market-data-preparation/required-clock-qualification.d.ts +43 -0
  12. package/dist/helpers/market-data-preparation/source-tape-dependencies.d.ts +19 -0
  13. package/dist/helpers/market-data-preparation/source-tape-operation.d.ts +109 -0
  14. package/dist/helpers/market-data-source-forensics.d.ts +400 -0
  15. package/dist/helpers/market-data-vendor-backfill/conformance-fixtures.d.ts +3 -5
  16. package/dist/helpers/market-data-vendor-backfill/contracts.d.ts +21 -56
  17. package/dist/helpers/market-data-vendor-backfill/core.d.ts +1 -0
  18. package/dist/helpers/market-data-vendor-backfill/cryptohftdata.d.ts +44 -5
  19. package/dist/helpers/market-data-vendor-backfill/manifests.d.ts +33 -1112
  20. package/dist/helpers/market-data-vendor-backfill/promotion.d.ts +1 -0
  21. package/dist/helpers/source-tape-sandbox.d.ts +82 -0
  22. package/dist/helpers/source-tape.d.ts +165 -0
  23. package/dist/index.js +220 -25
  24. package/dist/index.js.map +4 -4
  25. package/dist/market-data-preparation/fixtures/{conformance-v2.json → conformance-v3.json} +253 -56
  26. package/dist/market-data-preparation/policies/capability-policy.json +4 -4
  27. package/dist/market-data-preparation/policies/source-tape-capability-v1.json +35 -0
  28. package/dist/market-data-preparation/schema-manifest.json +30 -10
  29. package/dist/market-data-preparation/schemas/canonical-orderbook-export-request-v1.schema.json +5 -1
  30. package/dist/market-data-preparation/schemas/{canonical-orderbook-export-result-v1.schema.json → canonical-orderbook-export-result-v2.schema.json} +20 -6
  31. package/dist/market-data-preparation/schemas/order-book-depth-summary-parquet-projection-v1.schema.json +256 -0
  32. package/dist/market-data-preparation/schemas/order-book-levels-parquet-projection-v1.schema.json +220 -0
  33. package/dist/market-data-preparation/schemas/{preparation-product-pin-v1.schema.json → preparation-product-pin-v2.schema.json} +59 -9
  34. package/dist/market-data-preparation/schemas/promotion-receipt-v1.schema.json +5 -1
  35. package/dist/market-data-preparation/schemas/source-forensics-ledger-v1.schema.json +463 -0
  36. package/dist/market-data-preparation/schemas/source-qualification-record-v1.schema.json +394 -0
  37. package/dist/market-data-preparation.d.ts +7 -0
  38. package/dist/market-data-preparation.js +35448 -25240
  39. package/dist/market-data-preparation.js.map +118 -78
  40. package/dist/market-data-vendor-backfill.d.ts +7 -4
  41. package/dist/market-data-vendor-backfill.js +19338 -15503
  42. package/dist/market-data-vendor-backfill.js.map +65 -60
  43. package/package.json +1 -5
  44. package/dist/market-data-preparation/policies/capability-policy-v1.json +0 -28
  45. package/dist/market-data-preparation/policies/resource-policy-v1.json +0 -16
  46. package/dist/market-data-vendor-backfill/fixtures/conformance-v1.json +0 -448
  47. package/dist/market-data-vendor-backfill/policies/capability-policy-v2.json +0 -39
  48. package/dist/market-data-vendor-backfill/policies/capability-policy.json +0 -28
  49. package/dist/market-data-vendor-backfill/policies/resource-policy-v2.json +0 -16
  50. package/dist/market-data-vendor-backfill/policies/resource-policy.json +0 -16
  51. package/dist/market-data-vendor-backfill/schema-manifest.json +0 -31
  52. package/dist/market-data-vendor-backfill/schemas/archive-selection.schema.json +0 -224
  53. package/dist/market-data-vendor-backfill/schemas/promotion-receipt.schema.json +0 -171
  54. package/dist/market-data-vendor-backfill/schemas/request.schema.json +0 -166
  55. package/dist/market-data-vendor-backfill/schemas/required-clock.schema.json +0 -47
  56. package/dist/market-data-vendor-backfill/schemas/result.schema.json +0 -198
@@ -0,0 +1,19 @@
1
+ type ProjectionColumn = {
2
+ name: string;
3
+ physical_type: string;
4
+ logical_type: string | null;
5
+ nullable: boolean;
6
+ };
7
+ type ProjectionDocument = {
8
+ $id: string;
9
+ canonical_schema_id: string;
10
+ file_name: string;
11
+ parquet_metadata: {
12
+ key_value_metadata: unknown[];
13
+ };
14
+ columns: ProjectionColumn[];
15
+ };
16
+ export declare function assertParquetProjection(bytes: Uint8Array, projection: ProjectionDocument): void;
17
+ export declare function assertLevelsParquetProjection(bytes: Uint8Array): void;
18
+ export declare function assertDepthSummaryParquetProjection(bytes: Uint8Array): void;
19
+ export {};
@@ -7,6 +7,38 @@ export type DepositArchivePollerConfig = {
7
7
  lookbackMs: number;
8
8
  depositsLimit: number;
9
9
  };
10
+ type BinanceUnlockProgressState = "pending" | "credited_not_withdrawable" | "ok" | "failed" | "unknown";
11
+ type BinanceUnlockProgressQuality = "valid" | "unknown" | "contradictory";
12
+ declare const BINANCE_UNLOCK_PROGRESS_SOURCE: {
13
+ readonly venue: "binance";
14
+ readonly endpoint: "GET /sapi/v1/capital/deposit/hisrec";
15
+ readonly fields: {
16
+ readonly status: "info.status";
17
+ readonly confirmTimes: "info.confirmTimes";
18
+ readonly unlockConfirm: "info.unlockConfirm";
19
+ readonly completeTime: "info.completeTime";
20
+ };
21
+ };
22
+ export type BinanceUnlockProgress = {
23
+ version: 1;
24
+ state: BinanceUnlockProgressState;
25
+ progress_state: BinanceUnlockProgressQuality;
26
+ reason: string | null;
27
+ native_status: number | null;
28
+ current: number | null;
29
+ credit_required: number | null;
30
+ unlock_required: number | null;
31
+ complete_time: number | null;
32
+ observed_at: string;
33
+ source: typeof BINANCE_UNLOCK_PROGRESS_SOURCE;
34
+ };
35
+ /**
36
+ * Extracts Binance's native deposit confirmation fields without trusting
37
+ * ccxt's normalized status. `confirmTimes` is normally the venue string
38
+ * `"current/credit_required"`; the object form is accepted only for callers
39
+ * that already decoded that exact field.
40
+ */
41
+ export declare function parseBinanceUnlockProgress(record: Record<string, unknown>): BinanceUnlockProgress;
10
42
  /**
11
43
  * Advances the inclusive since-watermark only across a fully observed,
12
44
  * terminal prefix of deposit history.
@@ -31,3 +63,4 @@ export declare class DepositArchivePoller {
31
63
  stop(): Promise<void>;
32
64
  pollAllOnce(): Promise<boolean>;
33
65
  }
66
+ export {};
@@ -16,5 +16,5 @@ export declare function buildCanonicalOrderBookRows(input: {
16
16
  rawCapture: RawCapture;
17
17
  depthLimit: number;
18
18
  measurementBandsBps?: readonly number[];
19
- constructionMode?: OrderBookConstructionMode;
19
+ constructionMode?: OrderBookConstructionMode | "policy_neutral_top_n_state_change_tape/v1";
20
20
  }): CanonicalOrderBookRows;
@@ -1,16 +1,128 @@
1
1
  export declare const PREPARATION_CONFORMANCE_FIXTURES: Readonly<{
2
- fixture_id: "cex-market-data-preparation-conformance/v2";
3
- legacy_fixture_id: "market-data-vendor-backfill-conformance/v1";
2
+ fixture_id: "cex-market-data-preparation-conformance/v3";
4
3
  documents: Readonly<{
5
4
  backfill_result: import("./contracts").BackfillJobResultV2Wire;
6
5
  canonical_orderbook_export_request: import("./contracts").CanonicalOrderBookExportRequestWire;
7
6
  canonical_orderbook_export_result: import("./contracts").CanonicalOrderBookExportResultWire;
8
7
  preparation_product_pin: import("./contracts").PreparationProductPinWire;
8
+ library_operations: Readonly<{
9
+ required_clock_qualification: Readonly<{
10
+ operation_id: "market-data-required-clock-qualification/v1";
11
+ attempt_id: "arb-usdt-required-clock-conformance";
12
+ request: {
13
+ idempotency_key: string;
14
+ schema_id: string;
15
+ request_id: string;
16
+ attempt_id: string;
17
+ scope: {
18
+ exchange: string;
19
+ trading_pair: string;
20
+ market_type: "spot";
21
+ feed: "ORDERBOOK";
22
+ };
23
+ window: {
24
+ start_at: string;
25
+ end_at: string;
26
+ };
27
+ depth: number;
28
+ construction_mode: "sampled_top_n_snapshot";
29
+ source_policy: "authoritative_window";
30
+ target: {
31
+ environment: string;
32
+ cluster: string;
33
+ };
34
+ coverage_policy: {
35
+ policy_id: "prior-asof-strict/v1";
36
+ max_asof_lag_ms: number;
37
+ future_rows: "reject";
38
+ missing_required_event: "fail";
39
+ };
40
+ required_clock: {
41
+ schema_id: string;
42
+ clock_id: string;
43
+ file_name: string;
44
+ clock_sha256: string;
45
+ event_count: number;
46
+ };
47
+ initial_selection: import("../market-data-vendor-backfill/contracts").ArchiveSelectionWire;
48
+ expected_canonical_schema: {
49
+ schema_id: string;
50
+ schema_sha256: string;
51
+ };
52
+ product_pins: {
53
+ capability_policy: {
54
+ policy_id: "market-data-vendor-backfill-capabilities/v3";
55
+ policy_sha256: string;
56
+ };
57
+ resource_policy: {
58
+ policy_id: "market-data-vendor-backfill-resources/v2";
59
+ policy_sha256: string;
60
+ };
61
+ };
62
+ production_authorization_id: string;
63
+ };
64
+ required_clock: import("../market-data-vendor-backfill/contracts").RequiredClockWire;
65
+ artifacts: Readonly<{
66
+ ledger_file_name: "required-clock-ledger.json";
67
+ qualification_record_file_name: "required-clock-qualification.json";
68
+ }>;
69
+ }>;
70
+ source_tape: Readonly<{
71
+ operation_id: "market-data-source-tape/v1";
72
+ attempt_id: "arb-usdt-source-tape-conformance";
73
+ request_id: string;
74
+ scope: {
75
+ exchange: string;
76
+ trading_pair: string;
77
+ market_type: "spot";
78
+ feed: "ORDERBOOK";
79
+ };
80
+ window: {
81
+ start_at: string;
82
+ end_at: string;
83
+ };
84
+ depth: 100;
85
+ target: Readonly<{
86
+ environment: "sandbox";
87
+ cluster: "cex-archive-local";
88
+ }>;
89
+ production_authorization_id: string;
90
+ expected_canonical_schema: {
91
+ schema_id: string;
92
+ schema_sha256: string;
93
+ };
94
+ product_pins: Readonly<{
95
+ source_tape_capability: Readonly<{
96
+ policy_id: string;
97
+ policy_sha256: string;
98
+ }>;
99
+ resource_policy: Readonly<{
100
+ policy_id: "market-data-vendor-backfill-resources/v2";
101
+ policy_sha256: string;
102
+ }>;
103
+ adapter_policy: Readonly<{
104
+ policy_id: "cryptohftdata-orderbook-adapter/v1";
105
+ policy_sha256: string;
106
+ }>;
107
+ acquisition_policy: Readonly<{
108
+ policy_id: "cryptohftdata-hourly-acquisition/v2";
109
+ policy_sha256: string;
110
+ }>;
111
+ }>;
112
+ artifacts: Readonly<{
113
+ ledger_file_name: "source-tape-ledger.json";
114
+ qualification_record_file_name: "source-tape-qualification.json";
115
+ exporter_result_file_name: "source-tape-export-result.json";
116
+ }>;
117
+ }>;
118
+ }>;
9
119
  }>;
10
120
  identities: Readonly<{
11
121
  manifest_sha256: string;
12
122
  backfill_result_sha256: string;
13
123
  selection_sha256: string;
14
124
  export_result_sha256: string;
125
+ product_pin_sha256: string;
126
+ library_operations_sha256: string;
15
127
  }>;
16
128
  }>;