@usherlabs/cex-broker 0.2.49 → 0.2.51

package/dist/market-data-preparation/schemas/canonical-orderbook-export-request-v1.schema.json

@@ -25,7 +25,11 @@
 		},
 		"depth": { "type": "integer", "minimum": 1, "maximum": 500 },
 		"construction_mode": {
-			"enum": ["sampled_top_n_snapshot", "exact_l2_reconstruction"]
+			"enum": [
+				"sampled_top_n_snapshot",
+				"exact_l2_reconstruction",
+				"policy_neutral_top_n_state_change_tape/v1"
+			]
 		},
 		"canonical_schema_version": {
 			"type": "string",

package/dist/market-data-preparation/schemas/{canonical-orderbook-export-result-v1.schema.json → canonical-orderbook-export-result-v2.schema.json}

@@ -1,7 +1,7 @@
 {
 	"$schema": "https://json-schema.org/draft/2020-12/schema",
-	"$id": "https://schemas.usher.so/cex-canonical-orderbook-export-result/v1",
-	"title": "CEX canonical order-book exact-selection export result v1",
+	"$id": "https://schemas.usher.so/cex-canonical-orderbook-export-result/v2",
+	"title": "CEX canonical order-book exact-selection export result v2",
 	"type": "object",
 	"additionalProperties": false,
 	"required": [
@@ -16,7 +16,7 @@
 	],
 	"properties": {
 		"schema_id": {
-			"const": "https://schemas.usher.so/cex-canonical-orderbook-export-result/v1"
+			"const": "https://schemas.usher.so/cex-canonical-orderbook-export-result/v2"
 		},
 		"result_sha256": { "$ref": "#/$defs/sha256" },
 		"job_id": { "$ref": "#/$defs/uuid" },
@@ -176,7 +176,7 @@
 			],
 			"properties": {
 				"product_id": { "const": "cex-canonical-orderbook-export" },
-				"product_version": { "const": "cex-canonical-orderbook-export/v1" },
+				"product_version": { "const": "cex-canonical-orderbook-export/v2" },
 				"package": {
 					"type": "object",
 					"additionalProperties": false,
@@ -215,7 +215,14 @@
 		"artifact": {
 			"type": "object",
 			"additionalProperties": false,
-			"required": ["file_name", "rows", "bytes", "sha256"],
+			"required": [
+				"file_name",
+				"rows",
+				"bytes",
+				"sha256",
+				"projection_schema_id",
+				"projection_schema_sha256"
+			],
 			"properties": {
 				"file_name": {
 					"type": "string",
@@ -231,7 +238,14 @@
 					"minimum": 8,
 					"maximum": 9007199254740991
 				},
-				"sha256": { "$ref": "#/$defs/sha256" }
+				"sha256": { "$ref": "#/$defs/sha256" },
+				"projection_schema_id": {
+					"enum": [
+						"https://schemas.usher.so/cex-order-book-levels-parquet-projection/v1",
+						"https://schemas.usher.so/cex-order-book-depth-summary-parquet-projection/v1"
+					]
+				},
+				"projection_schema_sha256": { "$ref": "#/$defs/sha256" }
 			}
 		}
 	}

package/dist/market-data-preparation/schemas/order-book-depth-summary-parquet-projection-v1.schema.json

@@ -0,0 +1,256 @@
+{
+	"$schema": "https://json-schema.org/draft/2020-12/schema",
+	"$id": "https://schemas.usher.so/cex-order-book-depth-summary-parquet-projection/v1",
+	"title": "CEX order-book depth-summary Parquet capture-core projection v1",
+	"canonical_schema_id": "cex-order-book-canonical/v1",
+	"file_name": "order_book_depth_summary.parquet",
+	"parquet_metadata": {
+		"key_value_metadata": []
+	},
+	"columns": [
+		{
+			"name": "source",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "deployment_id",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "capture_bundle_id",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": true
+		},
+		{
+			"name": "exchange",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "symbol",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "trading_pair",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "source_symbol",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "asset_type",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "feed",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "provider",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "source_mode",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "capture_origin",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "source_time_ms",
+			"physical_type": "INT64",
+			"logical_type": "UINT_64",
+			"nullable": false
+		},
+		{
+			"name": "received_time_ms",
+			"physical_type": "INT64",
+			"logical_type": "UINT_64",
+			"nullable": false
+		},
+		{
+			"name": "raw_capture_id",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": true
+		},
+		{
+			"name": "raw_capture_scope",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": true
+		},
+		{
+			"name": "schema_version",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "checksum_algorithm",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "raw_checksum",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": true
+		},
+		{
+			"name": "provenance_complete",
+			"physical_type": "INT32",
+			"logical_type": "UINT_8",
+			"nullable": false
+		},
+		{
+			"name": "snapshot_id",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "construction_mode",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "gap_policy",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "depth_limit",
+			"physical_type": "INT32",
+			"logical_type": "UINT_16",
+			"nullable": false
+		},
+		{
+			"name": "sequence",
+			"physical_type": "INT64",
+			"logical_type": "UINT_64",
+			"nullable": true
+		},
+		{
+			"name": "exact_l2_reconstruction_complete",
+			"physical_type": "INT32",
+			"logical_type": "UINT_8",
+			"nullable": false
+		},
+		{
+			"name": "best_bid",
+			"physical_type": "FIXED_LEN_BYTE_ARRAY",
+			"logical_type": "DECIMAL(38,18)",
+			"nullable": false
+		},
+		{
+			"name": "best_ask",
+			"physical_type": "FIXED_LEN_BYTE_ARRAY",
+			"logical_type": "DECIMAL(38,18)",
+			"nullable": false
+		},
+		{
+			"name": "best_bid_amount",
+			"physical_type": "FIXED_LEN_BYTE_ARRAY",
+			"logical_type": "DECIMAL(38,18)",
+			"nullable": false
+		},
+		{
+			"name": "best_ask_amount",
+			"physical_type": "FIXED_LEN_BYTE_ARRAY",
+			"logical_type": "DECIMAL(38,18)",
+			"nullable": false
+		},
+		{
+			"name": "mid_price",
+			"physical_type": "FIXED_LEN_BYTE_ARRAY",
+			"logical_type": "DECIMAL(38,18)",
+			"nullable": false
+		},
+		{
+			"name": "spread",
+			"physical_type": "FIXED_LEN_BYTE_ARRAY",
+			"logical_type": "DECIMAL(38,18)",
+			"nullable": false
+		},
+		{
+			"name": "spread_bps",
+			"physical_type": "DOUBLE",
+			"logical_type": null,
+			"nullable": false
+		},
+		{
+			"name": "staleness_ms",
+			"physical_type": "INT64",
+			"logical_type": "UINT_64",
+			"nullable": false
+		},
+		{
+			"name": "bid_level_count",
+			"physical_type": "INT32",
+			"logical_type": "UINT_16",
+			"nullable": false
+		},
+		{
+			"name": "ask_level_count",
+			"physical_type": "INT32",
+			"logical_type": "UINT_16",
+			"nullable": false
+		},
+		{
+			"name": "measurement_bands_bps",
+			"physical_type": "LIST",
+			"logical_type": "LIST<UINT_32>",
+			"nullable": false
+		},
+		{
+			"name": "bid_depth_by_band",
+			"physical_type": "LIST",
+			"logical_type": "LIST<DECIMAL(38,18)>",
+			"nullable": false
+		},
+		{
+			"name": "ask_depth_by_band",
+			"physical_type": "LIST",
+			"logical_type": "LIST<DECIMAL(38,18)>",
+			"nullable": false
+		},
+		{
+			"name": "normalized_row_checksum",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		}
+	],
+	"metadata": {
+		"cex.canonical_schema_id": "cex-order-book-canonical/v1",
+		"cex.projection_schema_id": "https://schemas.usher.so/cex-order-book-depth-summary-parquet-projection/v1"
+	}
+}

package/dist/market-data-preparation/schemas/order-book-levels-parquet-projection-v1.schema.json

@@ -0,0 +1,220 @@
+{
+	"$schema": "https://json-schema.org/draft/2020-12/schema",
+	"$id": "https://schemas.usher.so/cex-order-book-levels-parquet-projection/v1",
+	"title": "CEX order-book levels Parquet capture-core projection v1",
+	"canonical_schema_id": "cex-order-book-canonical/v1",
+	"file_name": "order_book_levels.parquet",
+	"parquet_metadata": {
+		"key_value_metadata": []
+	},
+	"columns": [
+		{
+			"name": "source",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "deployment_id",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "capture_bundle_id",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": true
+		},
+		{
+			"name": "exchange",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "symbol",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "trading_pair",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "source_symbol",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "asset_type",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "feed",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "provider",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "source_mode",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "capture_origin",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "source_time_ms",
+			"physical_type": "INT64",
+			"logical_type": "UINT_64",
+			"nullable": false
+		},
+		{
+			"name": "received_time_ms",
+			"physical_type": "INT64",
+			"logical_type": "UINT_64",
+			"nullable": false
+		},
+		{
+			"name": "raw_capture_id",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": true
+		},
+		{
+			"name": "raw_capture_scope",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": true
+		},
+		{
+			"name": "schema_version",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "checksum_algorithm",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "raw_checksum",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": true
+		},
+		{
+			"name": "provenance_complete",
+			"physical_type": "INT32",
+			"logical_type": "UINT_8",
+			"nullable": false
+		},
+		{
+			"name": "snapshot_id",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "construction_mode",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "gap_policy",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "depth_limit",
+			"physical_type": "INT32",
+			"logical_type": "UINT_16",
+			"nullable": false
+		},
+		{
+			"name": "sequence",
+			"physical_type": "INT64",
+			"logical_type": "UINT_64",
+			"nullable": true
+		},
+		{
+			"name": "exact_l2_reconstruction_complete",
+			"physical_type": "INT32",
+			"logical_type": "UINT_8",
+			"nullable": false
+		},
+		{
+			"name": "side",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		},
+		{
+			"name": "level_index",
+			"physical_type": "INT32",
+			"logical_type": "UINT_16",
+			"nullable": false
+		},
+		{
+			"name": "price",
+			"physical_type": "FIXED_LEN_BYTE_ARRAY",
+			"logical_type": "DECIMAL(38,18)",
+			"nullable": false
+		},
+		{
+			"name": "amount",
+			"physical_type": "FIXED_LEN_BYTE_ARRAY",
+			"logical_type": "DECIMAL(38,18)",
+			"nullable": false
+		},
+		{
+			"name": "notional",
+			"physical_type": "FIXED_LEN_BYTE_ARRAY",
+			"logical_type": "DECIMAL(38,18)",
+			"nullable": false
+		},
+		{
+			"name": "mid_price",
+			"physical_type": "FIXED_LEN_BYTE_ARRAY",
+			"logical_type": "DECIMAL(38,18)",
+			"nullable": false
+		},
+		{
+			"name": "spread_from_mid_bps",
+			"physical_type": "DOUBLE",
+			"logical_type": null,
+			"nullable": false
+		},
+		{
+			"name": "normalized_row_checksum",
+			"physical_type": "BYTE_ARRAY",
+			"logical_type": "UTF8",
+			"nullable": false
+		}
+	],
+	"metadata": {
+		"cex.canonical_schema_id": "cex-order-book-canonical/v1",
+		"cex.projection_schema_id": "https://schemas.usher.so/cex-order-book-levels-parquet-projection/v1"
+	}
+}

package/dist/market-data-preparation/schemas/{preparation-product-pin-v1.schema.json → preparation-product-pin-v2.schema.json}

@@ -1,7 +1,7 @@
 {
 	"$schema": "https://json-schema.org/draft/2020-12/schema",
-	"$id": "cex-market-data-preparation-product-pin/v1",
-	"title": "CEX market-data preparation product pin v1",
+	"$id": "cex-market-data-preparation-product-pin/v2",
+	"title": "CEX market-data preparation product pin v2",
 	"type": "object",
 	"additionalProperties": false,
 	"required": [
@@ -11,10 +11,12 @@
 		"schema_manifest",
 		"schema_pins",
 		"capability_policy",
-		"resource_policy"
+		"source_tape_capability",
+		"resource_policy",
+		"preparation_library"
 	],
 	"properties": {
-		"schema_id": { "const": "cex-market-data-preparation-product-pin/v1" },
+		"schema_id": { "const": "cex-market-data-preparation-product-pin/v2" },
 		"package": {
 			"type": "object",
 			"additionalProperties": false,
@@ -64,7 +66,7 @@
 					"product_version": {
 						"enum": [
 							"market-data-vendor-backfill/v1",
-							"cex-canonical-orderbook-export/v1"
+							"cex-canonical-orderbook-export/v2"
 						]
 					},
 					"relative_path": {
@@ -81,7 +83,7 @@
 			"required": ["schema_id", "manifest_sha256", "relative_path"],
 			"properties": {
 				"schema_id": {
-					"const": "https://schemas.usher.so/market-data-vendor-backfill-schema-manifest/v2"
+					"const": "https://schemas.usher.so/market-data-vendor-backfill-schema-manifest/v3"
 				},
 				"manifest_sha256": { "$ref": "#/$defs/sha256" },
 				"relative_path": {
@@ -91,8 +93,8 @@
 		},
 		"schema_pins": {
 			"type": "array",
-			"minItems": 8,
-			"maxItems": 8,
+			"minItems": 12,
+			"maxItems": 12,
 			"uniqueItems": true,
 			"items": {
 				"type": "object",
@@ -105,7 +107,55 @@
 			}
 		},
 		"capability_policy": { "$ref": "#/$defs/policyPin" },
-		"resource_policy": { "$ref": "#/$defs/policyPin" }
+		"source_tape_capability": { "$ref": "#/$defs/policyPin" },
+		"resource_policy": { "$ref": "#/$defs/policyPin" },
+		"preparation_library": {
+			"type": "object",
+			"additionalProperties": false,
+			"required": [
+				"exported_subpath",
+				"runtime_entry_sha256",
+				"declaration_sha256",
+				"operations"
+			],
+			"properties": {
+				"exported_subpath": {
+					"const": "@usherlabs/cex-broker/market-data-preparation"
+				},
+				"runtime_entry_sha256": { "$ref": "#/$defs/sha256" },
+				"declaration_sha256": { "$ref": "#/$defs/sha256" },
+				"operations": {
+					"type": "array",
+					"prefixItems": [
+						{
+							"type": "object",
+							"additionalProperties": false,
+							"required": ["symbol", "operation_id"],
+							"properties": {
+								"symbol": { "const": "runMarketDataSourceTape" },
+								"operation_id": { "const": "market-data-source-tape/v1" }
+							}
+						},
+						{
+							"type": "object",
+							"additionalProperties": false,
+							"required": ["symbol", "operation_id"],
+							"properties": {
+								"symbol": {
+									"const": "runMarketDataRequiredClockQualification"
+								},
+								"operation_id": {
+									"const": "market-data-required-clock-qualification/v1"
+								}
+							}
+						}
+					],
+					"items": false,
+					"minItems": 2,
+					"maxItems": 2
+				}
+			}
+		}
 	},
 	"$defs": {
 		"sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },

package/dist/market-data-preparation/schemas/promotion-receipt-v1.schema.json

@@ -68,7 +68,11 @@
 		"window": { "$ref": "#/$defs/interval" },
 		"depth": { "type": "integer", "minimum": 1, "maximum": 500 },
 		"construction_mode": {
-			"enum": ["sampled_top_n_snapshot", "exact_l2_reconstruction"]
+			"enum": [
+				"sampled_top_n_snapshot",
+				"exact_l2_reconstruction",
+				"policy_neutral_top_n_state_change_tape/v1"
+			]
 		},
 		"canonical_schema": {
 			"type": "object",