@tungthedev/streams-server 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CODE_OF_CONDUCT.md +45 -0
- package/CONTRIBUTING.md +76 -0
- package/LICENSE +201 -0
- package/README.md +58 -0
- package/SECURITY.md +42 -0
- package/bin/prisma-streams-server +2 -0
- package/package.json +46 -0
- package/src/app.ts +583 -0
- package/src/app_core.ts +3144 -0
- package/src/app_local.ts +206 -0
- package/src/auth.ts +124 -0
- package/src/auto_tune.ts +69 -0
- package/src/backpressure.ts +66 -0
- package/src/bootstrap.ts +613 -0
- package/src/compute/demo_entry.ts +415 -0
- package/src/compute/demo_site.ts +1242 -0
- package/src/compute/entry.ts +19 -0
- package/src/compute/package_entry.ts +4 -0
- package/src/compute/virtual-modules.d.ts +15 -0
- package/src/compute/worker_module_url.ts +9 -0
- package/src/concurrency_gate.ts +108 -0
- package/src/config.ts +402 -0
- package/src/db/bootstrap_store.ts +9 -0
- package/src/db/db.ts +2424 -0
- package/src/db/schema.ts +925 -0
- package/src/db/sqlite_manifest_snapshot.ts +81 -0
- package/src/db/sqlite_touch_store.ts +491 -0
- package/src/db/sqlite_wal_store.ts +472 -0
- package/src/details/full_mode_details.ts +568 -0
- package/src/expiry_sweeper.ts +47 -0
- package/src/foreground_activity.ts +55 -0
- package/src/hist.ts +169 -0
- package/src/index/binary_fuse.ts +379 -0
- package/src/index/indexer.ts +947 -0
- package/src/index/lexicon_file_cache.ts +261 -0
- package/src/index/lexicon_format.ts +93 -0
- package/src/index/lexicon_indexer.ts +863 -0
- package/src/index/run_cache.ts +84 -0
- package/src/index/run_format.ts +213 -0
- package/src/index/schedule.ts +28 -0
- package/src/index/secondary_indexer.ts +901 -0
- package/src/index/secondary_schema.ts +105 -0
- package/src/ingest.ts +309 -0
- package/src/lens/lens.ts +501 -0
- package/src/manifest.ts +249 -0
- package/src/memory.ts +334 -0
- package/src/metrics.ts +147 -0
- package/src/metrics_emitter.ts +83 -0
- package/src/notifier.ts +180 -0
- package/src/objectstore/accounting.ts +151 -0
- package/src/objectstore/interface.ts +13 -0
- package/src/objectstore/mock_r2.ts +269 -0
- package/src/objectstore/null.ts +32 -0
- package/src/objectstore/r2.ts +318 -0
- package/src/observe/pairing.ts +61 -0
- package/src/observe/request.ts +772 -0
- package/src/offset.ts +70 -0
- package/src/postgres/bootstrap.ts +269 -0
- package/src/postgres/companions.ts +197 -0
- package/src/postgres/control_restore.ts +109 -0
- package/src/postgres/details.ts +189 -0
- package/src/postgres/lexicon_index.ts +260 -0
- package/src/postgres/routing_index.ts +189 -0
- package/src/postgres/rows.ts +132 -0
- package/src/postgres/schema.ts +355 -0
- package/src/postgres/secondary_index.ts +238 -0
- package/src/postgres/segments.ts +900 -0
- package/src/postgres/stats.ts +103 -0
- package/src/postgres/store.ts +947 -0
- package/src/postgres/touch.ts +591 -0
- package/src/postgres/types.ts +32 -0
- package/src/profiles/evlog/schema.ts +234 -0
- package/src/profiles/evlog.ts +473 -0
- package/src/profiles/generic.ts +51 -0
- package/src/profiles/index.ts +237 -0
- package/src/profiles/metrics/block_format.ts +109 -0
- package/src/profiles/metrics/normalize.ts +366 -0
- package/src/profiles/metrics/schema.ts +319 -0
- package/src/profiles/metrics.ts +83 -0
- package/src/profiles/otelTraces/normalize.ts +955 -0
- package/src/profiles/otelTraces/otlp.ts +1002 -0
- package/src/profiles/otelTraces/schema.ts +408 -0
- package/src/profiles/otelTraces.ts +390 -0
- package/src/profiles/profile.ts +284 -0
- package/src/profiles/stateProtocol/change_event_conformance.typecheck.ts +35 -0
- package/src/profiles/stateProtocol/changes.ts +24 -0
- package/src/profiles/stateProtocol/ingest.ts +115 -0
- package/src/profiles/stateProtocol/routes.ts +511 -0
- package/src/profiles/stateProtocol/types.ts +6 -0
- package/src/profiles/stateProtocol/validation.ts +51 -0
- package/src/profiles/stateProtocol.ts +107 -0
- package/src/read_filter.ts +468 -0
- package/src/reader.ts +2986 -0
- package/src/runtime/hash.ts +156 -0
- package/src/runtime/hash_vendor/LICENSE.hash-wasm +38 -0
- package/src/runtime/hash_vendor/NOTICE.md +8 -0
- package/src/runtime/hash_vendor/xxhash3.umd.min.cjs +7 -0
- package/src/runtime/hash_vendor/xxhash32.umd.min.cjs +7 -0
- package/src/runtime/hash_vendor/xxhash64.umd.min.cjs +7 -0
- package/src/runtime/host_runtime.ts +5 -0
- package/src/runtime_memory.ts +200 -0
- package/src/runtime_memory_sampler.ts +237 -0
- package/src/schema/lens_schema.ts +290 -0
- package/src/schema/proof.ts +547 -0
- package/src/schema/read_json.ts +51 -0
- package/src/schema/registry.ts +966 -0
- package/src/search/agg_format.ts +638 -0
- package/src/search/aggregate.ts +409 -0
- package/src/search/binary/codec.ts +162 -0
- package/src/search/binary/docset.ts +67 -0
- package/src/search/binary/restart_strings.ts +181 -0
- package/src/search/binary/varint.ts +34 -0
- package/src/search/bitset.ts +19 -0
- package/src/search/col_format.ts +382 -0
- package/src/search/col_runtime.ts +59 -0
- package/src/search/column_encoding.ts +43 -0
- package/src/search/companion_file_cache.ts +319 -0
- package/src/search/companion_format.ts +327 -0
- package/src/search/companion_manager.ts +1305 -0
- package/src/search/companion_plan.ts +229 -0
- package/src/search/exact_format.ts +281 -0
- package/src/search/exact_runtime.ts +55 -0
- package/src/search/fts_format.ts +423 -0
- package/src/search/fts_runtime.ts +333 -0
- package/src/search/query.ts +875 -0
- package/src/search/schema.ts +245 -0
- package/src/segment/cache.ts +270 -0
- package/src/segment/cached_segment.ts +89 -0
- package/src/segment/format.ts +403 -0
- package/src/segment/segmenter.ts +412 -0
- package/src/segment/segmenter_worker.ts +72 -0
- package/src/segment/segmenter_workers.ts +130 -0
- package/src/server.ts +264 -0
- package/src/server_auto_tune.ts +158 -0
- package/src/sqlite/adapter.ts +335 -0
- package/src/sqlite/runtime_stats.ts +163 -0
- package/src/stats.ts +205 -0
- package/src/store/append.ts +50 -0
- package/src/store/bootstrap_restore_store.ts +71 -0
- package/src/store/capabilities.ts +86 -0
- package/src/store/full_mode_details_store.ts +71 -0
- package/src/store/index_store.ts +104 -0
- package/src/store/profile_touch_store.ts +1 -0
- package/src/store/rows.ts +144 -0
- package/src/store/schema_profile_store.ts +73 -0
- package/src/store/schema_publication.ts +6 -0
- package/src/store/segment_manifest_store.ts +129 -0
- package/src/store/segment_read_store.ts +22 -0
- package/src/store/stats_accounting_store.ts +83 -0
- package/src/store/touch_store.ts +98 -0
- package/src/store/wal_store.ts +21 -0
- package/src/stream_size_reconciler.ts +100 -0
- package/src/touch/canonical_change.ts +7 -0
- package/src/touch/live_keys.ts +158 -0
- package/src/touch/live_metrics.ts +841 -0
- package/src/touch/live_templates.ts +449 -0
- package/src/touch/manager.ts +1292 -0
- package/src/touch/process_batch.ts +576 -0
- package/src/touch/processor_worker.ts +85 -0
- package/src/touch/spec.ts +459 -0
- package/src/touch/touch_journal.ts +771 -0
- package/src/touch/touch_key_id.ts +20 -0
- package/src/touch/worker_pool.ts +191 -0
- package/src/touch/worker_protocol.ts +57 -0
- package/src/types/proper-lockfile.d.ts +1 -0
- package/src/uploader.ts +358 -0
- package/src/util/base32_crockford.ts +81 -0
- package/src/util/bloom256.ts +67 -0
- package/src/util/byte_lru.ts +73 -0
- package/src/util/cleanup.ts +22 -0
- package/src/util/crc32c.ts +29 -0
- package/src/util/ds_error.ts +15 -0
- package/src/util/duration.ts +17 -0
- package/src/util/endian.ts +53 -0
- package/src/util/json_pointer.ts +148 -0
- package/src/util/log.ts +25 -0
- package/src/util/lru.ts +53 -0
- package/src/util/retry.ts +35 -0
- package/src/util/siphash.ts +71 -0
- package/src/util/stream_paths.ts +50 -0
- package/src/util/time.ts +14 -0
- package/src/util/yield.ts +3 -0
- package/src/util/zstd.ts +24 -0
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { Result } from "better-result";
|
|
2
|
+
import type {
|
|
3
|
+
StreamProfileDefinition,
|
|
4
|
+
StreamProfilePersistResult,
|
|
5
|
+
StreamProfileReadResult,
|
|
6
|
+
} from "./profile";
|
|
7
|
+
import { cloneStreamProfileSpec, expectPlainObjectResult, rejectUnknownKeysResult, normalizeProfileContentType } from "./profile";
|
|
8
|
+
import { buildInternalMetricsRegistry, buildMetricsDefaultRegistry } from "./metrics/schema";
|
|
9
|
+
import { normalizeMetricsRecordResult } from "./metrics/normalize";
|
|
10
|
+
|
|
11
|
+
const INTERNAL_METRICS_STREAM = "__stream_metrics__";
|
|
12
|
+
|
|
13
|
+
export type MetricsStreamProfile = {
|
|
14
|
+
kind: "metrics";
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
function cloneMetricsProfile(): MetricsStreamProfile {
|
|
18
|
+
return { kind: "metrics" };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function validateMetricsProfileResult(raw: unknown, path: string): Result<MetricsStreamProfile, { message: string }> {
|
|
22
|
+
const objRes = expectPlainObjectResult(raw, path);
|
|
23
|
+
if (Result.isError(objRes)) return objRes;
|
|
24
|
+
if (objRes.value.kind !== "metrics") return Result.err({ message: `${path}.kind must be metrics` });
|
|
25
|
+
const keyCheck = rejectUnknownKeysResult(objRes.value, ["kind"], path);
|
|
26
|
+
if (Result.isError(keyCheck)) return keyCheck;
|
|
27
|
+
return Result.ok(cloneMetricsProfile());
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export const METRICS_STREAM_PROFILE_DEFINITION: StreamProfileDefinition = {
|
|
31
|
+
kind: "metrics",
|
|
32
|
+
usesStoredProfileRow: false,
|
|
33
|
+
|
|
34
|
+
defaultProfile(): MetricsStreamProfile {
|
|
35
|
+
return cloneMetricsProfile();
|
|
36
|
+
},
|
|
37
|
+
|
|
38
|
+
validateResult(raw, path) {
|
|
39
|
+
return validateMetricsProfileResult(raw, path);
|
|
40
|
+
},
|
|
41
|
+
|
|
42
|
+
readProfileResult(): Result<StreamProfileReadResult, { message: string }> {
|
|
43
|
+
return Result.ok({ profile: cloneMetricsProfile(), cache: null });
|
|
44
|
+
},
|
|
45
|
+
|
|
46
|
+
persistProfileResult({ stream, streamRow, profile }): Result<StreamProfilePersistResult, { kind: "bad_request"; message: string }> {
|
|
47
|
+
if (profile.kind !== "metrics") return Result.err({ kind: "bad_request", message: "invalid metrics profile" });
|
|
48
|
+
const contentType = normalizeProfileContentType(streamRow.content_type);
|
|
49
|
+
if (contentType !== "application/json") {
|
|
50
|
+
return Result.err({
|
|
51
|
+
kind: "bad_request",
|
|
52
|
+
message: "metrics profile requires application/json stream content-type",
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
const desiredRegistry =
|
|
56
|
+
stream === INTERNAL_METRICS_STREAM ? buildInternalMetricsRegistry(stream) : buildMetricsDefaultRegistry(stream);
|
|
57
|
+
return Result.ok({
|
|
58
|
+
profile: cloneStreamProfileSpec(cloneMetricsProfile()),
|
|
59
|
+
cache: null,
|
|
60
|
+
schemaRegistry: desiredRegistry,
|
|
61
|
+
streamProfile: "metrics",
|
|
62
|
+
profileJson: null,
|
|
63
|
+
touchState: streamRow.profile === "state-protocol" ? "delete" : "preserve",
|
|
64
|
+
});
|
|
65
|
+
},
|
|
66
|
+
|
|
67
|
+
jsonIngest: {
|
|
68
|
+
prepareRecordResult({ value }) {
|
|
69
|
+
const normalizedRes = normalizeMetricsRecordResult(value);
|
|
70
|
+
if (Result.isError(normalizedRes)) return normalizedRes;
|
|
71
|
+
return Result.ok({
|
|
72
|
+
value: normalizedRes.value.value,
|
|
73
|
+
routingKey: normalizedRes.value.routingKey,
|
|
74
|
+
});
|
|
75
|
+
},
|
|
76
|
+
},
|
|
77
|
+
|
|
78
|
+
metrics: {
|
|
79
|
+
normalizeRecordResult({ value }) {
|
|
80
|
+
return normalizeMetricsRecordResult(value);
|
|
81
|
+
},
|
|
82
|
+
},
|
|
83
|
+
};
|