@adcp/sdk 16.0.0-beta.3 → 16.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/auth/oauth/diagnose.js +3 -11
- package/dist/lib/auth/oauth/diagnose.mjs +4 -12
- package/dist/lib/auth/oauth/diagnostics.d.mts +2 -1
- package/dist/lib/auth/oauth/diagnostics.d.ts +2 -1
- package/dist/lib/auth/oauth/diagnostics.js +8 -10
- package/dist/lib/auth/oauth/diagnostics.mjs +7 -10
- package/dist/lib/index.d.mts +1 -1
- package/dist/lib/index.d.ts +1 -1
- package/dist/lib/index.js +2 -0
- package/dist/lib/index.mjs +7 -1
- package/dist/lib/reporting/consumer-runtime.js +2 -1
- package/dist/lib/reporting/consumer-runtime.mjs +2 -1
- package/dist/lib/reporting/ledger/handler.js +50 -11
- package/dist/lib/reporting/ledger/handler.mjs +50 -12
- package/dist/lib/reporting/ledger/index.d.mts +17 -14
- package/dist/lib/reporting/ledger/index.d.ts +17 -14
- package/dist/lib/reporting/ledger/index.js +32 -20
- package/dist/lib/reporting/ledger/index.mjs +21 -10
- package/dist/lib/reporting/ledger/offering-validation.d.mts +9 -0
- package/dist/lib/reporting/ledger/offering-validation.d.ts +9 -0
- package/dist/lib/reporting/{gcs/bucket-policy.js → ledger/offering-validation.js} +9 -8
- package/dist/lib/reporting/ledger/offering-validation.mjs +8 -0
- package/dist/lib/reporting/ledger/receipts.d.mts +48 -0
- package/dist/lib/reporting/ledger/receipts.d.ts +48 -0
- package/dist/lib/reporting/ledger/receipts.js +186 -0
- package/dist/lib/reporting/ledger/receipts.mjs +161 -0
- package/dist/lib/reporting/ledger/status-ingest.js +3 -3
- package/dist/lib/reporting/ledger/status-ingest.mjs +4 -5
- package/dist/lib/reporting/ledger/types.d.mts +43 -0
- package/dist/lib/reporting/ledger/types.d.ts +43 -0
- package/dist/lib/reporting/ledger/types.js +31 -2
- package/dist/lib/reporting/ledger/types.mjs +26 -1
- package/dist/lib/reporting/pagination.d.mts +6 -0
- package/dist/lib/reporting/pagination.d.ts +6 -0
- package/dist/lib/reporting/{gcs/errors.js → pagination.js} +12 -17
- package/dist/lib/reporting/pagination.mjs +12 -0
- package/dist/lib/reporting/reconciliation.d.mts +1 -0
- package/dist/lib/reporting/reconciliation.d.ts +1 -0
- package/dist/lib/reporting/reconciliation.js +3 -2
- package/dist/lib/reporting/reconciliation.mjs +3 -2
- package/dist/lib/reporting/service/index.d.mts +79 -153
- package/dist/lib/reporting/service/index.d.ts +79 -153
- package/dist/lib/reporting/service/index.js +34 -491
- package/dist/lib/reporting/service/index.mjs +36 -510
- package/dist/lib/reporting/service/scheduler-support.d.mts +43 -0
- package/dist/lib/reporting/service/scheduler-support.d.ts +43 -0
- package/dist/lib/reporting/service/scheduler-support.js +125 -0
- package/dist/lib/reporting/service/scheduler-support.mjs +96 -0
- package/dist/lib/schemas-data/v2.5/_provenance.json +1 -1
- package/dist/lib/signing/canonicalize.d.mts +4 -0
- package/dist/lib/signing/canonicalize.d.ts +4 -0
- package/dist/lib/signing/canonicalize.js +14 -8
- package/dist/lib/signing/canonicalize.mjs +14 -8
- package/dist/lib/signing/content-digest.d.mts +8 -0
- package/dist/lib/signing/content-digest.d.ts +8 -0
- package/dist/lib/signing/content-digest.js +11 -5
- package/dist/lib/signing/content-digest.mjs +10 -5
- package/dist/lib/signing/signer.js +9 -6
- package/dist/lib/signing/signer.mjs +10 -7
- package/dist/lib/signing/web-bot-auth.js +103 -37
- package/dist/lib/signing/web-bot-auth.mjs +108 -37
- package/dist/lib/testing/storyboard/junit.js +16 -3
- package/dist/lib/testing/storyboard/junit.mjs +16 -3
- package/dist/lib/testing/storyboard/request-signing/builder.js +10 -9
- package/dist/lib/testing/storyboard/request-signing/builder.mjs +10 -9
- package/dist/lib/testing/storyboard/runner.js +94 -25
- package/dist/lib/testing/storyboard/runner.mjs +94 -25
- package/dist/lib/testing/storyboard/types.d.mts +6 -0
- package/dist/lib/testing/storyboard/types.d.ts +6 -0
- package/dist/lib/testing/storyboard/validations.d.mts +1 -1
- package/dist/lib/testing/storyboard/validations.d.ts +1 -1
- package/dist/lib/version.d.mts +3 -3
- package/dist/lib/version.d.ts +3 -3
- package/dist/lib/version.js +3 -3
- package/dist/lib/version.mjs +3 -3
- package/docs/TYPE-SUMMARY.md +23 -5
- package/docs/guides/REPORTING-GCS-FENCE.md +3 -3
- package/docs/guides/REPORTING-GCS-MANAGED.md +4 -4
- package/docs/guides/REPORTING-LEDGER-CUTOVER.md +7 -7
- package/docs/guides/REPORTING-LEDGER.md +107 -51
- package/docs/guides/REPORTING-OPERATIONS.md +66 -5
- package/docs/guides/REPORTING-STABILITY.md +36 -32
- package/docs/llms.txt +4 -3
- package/docs/migration-14.x-rc-worksheet.md +4 -7
- package/docs/migration-reporting-storage-package.md +186 -0
- package/examples/README.md +1 -1
- package/package.json +13 -78
- package/dist/lib/reporting/azure/index.d.mts +0 -31
- package/dist/lib/reporting/azure/index.d.ts +0 -31
- package/dist/lib/reporting/azure/index.js +0 -266
- package/dist/lib/reporting/azure/index.mjs +0 -249
- package/dist/lib/reporting/bigquery/index.d.mts +0 -246
- package/dist/lib/reporting/bigquery/index.d.ts +0 -246
- package/dist/lib/reporting/bigquery/index.js +0 -633
- package/dist/lib/reporting/bigquery/index.mjs +0 -594
- package/dist/lib/reporting/gcs/adapter.d.mts +0 -38
- package/dist/lib/reporting/gcs/adapter.d.ts +0 -38
- package/dist/lib/reporting/gcs/adapter.js +0 -342
- package/dist/lib/reporting/gcs/adapter.mjs +0 -325
- package/dist/lib/reporting/gcs/bucket-policy.d.mts +0 -3
- package/dist/lib/reporting/gcs/bucket-policy.d.ts +0 -3
- package/dist/lib/reporting/gcs/bucket-policy.mjs +0 -7
- package/dist/lib/reporting/gcs/errors.d.mts +0 -11
- package/dist/lib/reporting/gcs/errors.d.ts +0 -11
- package/dist/lib/reporting/gcs/errors.mjs +0 -16
- package/dist/lib/reporting/gcs/fence.d.mts +0 -41
- package/dist/lib/reporting/gcs/fence.d.ts +0 -41
- package/dist/lib/reporting/gcs/fence.js +0 -250
- package/dist/lib/reporting/gcs/fence.mjs +0 -230
- package/dist/lib/reporting/gcs/index.d.mts +0 -18
- package/dist/lib/reporting/gcs/index.d.ts +0 -18
- package/dist/lib/reporting/gcs/index.js +0 -44
- package/dist/lib/reporting/gcs/index.mjs +0 -14
- package/dist/lib/reporting/gcs/io.d.mts +0 -7
- package/dist/lib/reporting/gcs/io.d.ts +0 -7
- package/dist/lib/reporting/gcs/io.js +0 -95
- package/dist/lib/reporting/gcs/io.mjs +0 -69
- package/dist/lib/reporting/gcs/reader.d.mts +0 -67
- package/dist/lib/reporting/gcs/reader.d.ts +0 -67
- package/dist/lib/reporting/gcs/reader.js +0 -193
- package/dist/lib/reporting/gcs/reader.mjs +0 -169
- package/dist/lib/reporting/gcs/row-provider.d.mts +0 -39
- package/dist/lib/reporting/gcs/row-provider.d.ts +0 -39
- package/dist/lib/reporting/gcs/row-provider.js +0 -225
- package/dist/lib/reporting/gcs/row-provider.mjs +0 -208
- package/dist/lib/reporting/gcs/validation.d.mts +0 -5
- package/dist/lib/reporting/gcs/validation.d.ts +0 -5
- package/dist/lib/reporting/gcs/validation.js +0 -49
- package/dist/lib/reporting/gcs/validation.mjs +0 -25
- package/dist/lib/reporting/ledger/change-feed.d.mts +0 -69
- package/dist/lib/reporting/ledger/change-feed.d.ts +0 -69
- package/dist/lib/reporting/ledger/change-feed.js +0 -107
- package/dist/lib/reporting/ledger/change-feed.mjs +0 -80
- package/dist/lib/reporting/ledger/managed-postgres.d.mts +0 -293
- package/dist/lib/reporting/ledger/managed-postgres.d.ts +0 -293
- package/dist/lib/reporting/ledger/managed-postgres.js +0 -2346
- package/dist/lib/reporting/ledger/managed-postgres.mjs +0 -2332
- package/dist/lib/reporting/ledger/managed-router.d.mts +0 -148
- package/dist/lib/reporting/ledger/managed-router.d.ts +0 -148
- package/dist/lib/reporting/ledger/managed-router.js +0 -280
- package/dist/lib/reporting/ledger/managed-router.mjs +0 -258
- package/dist/lib/reporting/ledger/managed.d.mts +0 -535
- package/dist/lib/reporting/ledger/managed.d.ts +0 -535
- package/dist/lib/reporting/ledger/managed.js +0 -761
- package/dist/lib/reporting/ledger/managed.mjs +0 -729
- package/dist/lib/reporting/ledger/notification-activity.d.mts +0 -332
- package/dist/lib/reporting/ledger/notification-activity.d.ts +0 -332
- package/dist/lib/reporting/ledger/notification-activity.js +0 -1366
- package/dist/lib/reporting/ledger/notification-activity.mjs +0 -1337
- package/dist/lib/reporting/ledger/object-writes.d.mts +0 -128
- package/dist/lib/reporting/ledger/object-writes.d.ts +0 -128
- package/dist/lib/reporting/ledger/object-writes.js +0 -147
- package/dist/lib/reporting/ledger/object-writes.mjs +0 -114
- package/dist/lib/reporting/ledger/postgres.d.mts +0 -689
- package/dist/lib/reporting/ledger/postgres.d.ts +0 -689
- package/dist/lib/reporting/ledger/postgres.js +0 -4462
- package/dist/lib/reporting/ledger/postgres.mjs +0 -4462
- package/dist/lib/reporting/ledger/row-storage-migration.d.mts +0 -59
- package/dist/lib/reporting/ledger/row-storage-migration.d.ts +0 -59
- package/dist/lib/reporting/ledger/row-storage-migration.js +0 -365
- package/dist/lib/reporting/ledger/row-storage-migration.mjs +0 -339
- package/dist/lib/reporting/ledger/row-storage-object.d.mts +0 -203
- package/dist/lib/reporting/ledger/row-storage-object.d.ts +0 -203
- package/dist/lib/reporting/ledger/row-storage-object.js +0 -379
- package/dist/lib/reporting/ledger/row-storage-object.mjs +0 -335
- package/dist/lib/reporting/ledger/row-storage-provider-io.d.mts +0 -44
- package/dist/lib/reporting/ledger/row-storage-provider-io.d.ts +0 -44
- package/dist/lib/reporting/ledger/row-storage-provider-io.js +0 -142
- package/dist/lib/reporting/ledger/row-storage-provider-io.mjs +0 -111
- package/dist/lib/reporting/ledger/row-storage.d.mts +0 -316
- package/dist/lib/reporting/ledger/row-storage.d.ts +0 -316
- package/dist/lib/reporting/ledger/row-storage.js +0 -812
- package/dist/lib/reporting/ledger/row-storage.mjs +0 -802
- package/dist/lib/reporting/s3/index.d.mts +0 -34
- package/dist/lib/reporting/s3/index.d.ts +0 -34
- package/dist/lib/reporting/s3/index.js +0 -347
- package/dist/lib/reporting/s3/index.mjs +0 -340
- package/dist/lib/reporting/service/maintenance.d.mts +0 -148
- package/dist/lib/reporting/service/maintenance.d.ts +0 -148
- package/dist/lib/reporting/service/maintenance.js +0 -185
- package/dist/lib/reporting/service/maintenance.mjs +0 -159
- package/examples/reliable-reporting-service/README.md +0 -169
|
@@ -1,812 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __defProp = Object.defineProperty;
|
|
3
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
-
var __export = (target, all) => {
|
|
7
|
-
for (var name in all)
|
|
8
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
-
};
|
|
10
|
-
var __copyProps = (to, from, except, desc) => {
|
|
11
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
-
for (let key of __getOwnPropNames(from))
|
|
13
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
-
}
|
|
16
|
-
return to;
|
|
17
|
-
};
|
|
18
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
-
var row_storage_exports = {};
|
|
20
|
-
__export(row_storage_exports, {
|
|
21
|
-
ReportingRowStorageV1: () => ReportingRowStorageV1,
|
|
22
|
-
ReportingRowStoreError: () => import_row_storage_errors2.ReportingRowStoreError,
|
|
23
|
-
isReportingRowStoreError: () => import_row_storage_errors2.isReportingRowStoreError,
|
|
24
|
-
reportingRowWriteIntentIdV1: () => reportingRowWriteIntentIdV1
|
|
25
|
-
});
|
|
26
|
-
module.exports = __toCommonJS(row_storage_exports);
|
|
27
|
-
var import_crypto = require("crypto");
|
|
28
|
-
var import_zlib = require("zlib");
|
|
29
|
-
var import_jcs = require('../../utils/jcs.js');
|
|
30
|
-
var import_row_encoding = require('./row-encoding.js');
|
|
31
|
-
var import_row_storage_migration = require('./row-storage-migration.js');
|
|
32
|
-
var import_row_storage_errors = require('./row-storage-errors.js');
|
|
33
|
-
var import_row_storage_object = require('./row-storage-object.js');
|
|
34
|
-
var import_row_storage_errors2 = require('./row-storage-errors.js');
|
|
35
|
-
const DEFAULT_BINDING_ID = "postgres";
|
|
36
|
-
const BINDING_ID = /^[A-Za-z0-9_.:-]{1,128}$/;
|
|
37
|
-
const CREDENTIAL_REF = /^[A-Za-z0-9_.:-]{1,128}$/;
|
|
38
|
-
const MAX_ADOPTED_OBJECT_BYTES = 2 * import_row_encoding.REPORTING_ROW_CHUNK_MAX_BYTES + 1024 * 1024;
|
|
39
|
-
class ReportingRowStorageV1 {
|
|
40
|
-
constructor(pool, options = {}) {
|
|
41
|
-
this.pool = pool;
|
|
42
|
-
this.options = options;
|
|
43
|
-
const bindings = options.bindings ?? { [DEFAULT_BINDING_ID]: { kind: "postgres" } };
|
|
44
|
-
this.providers = Object.freeze({ ...options.providers ?? {} });
|
|
45
|
-
const ids = Object.keys(bindings);
|
|
46
|
-
if (ids.length === 0) throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", "at least one row binding is required");
|
|
47
|
-
for (const id of ids) {
|
|
48
|
-
if (!BINDING_ID.test(id)) throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", `invalid row binding ID ${id}`);
|
|
49
|
-
const binding = bindings[id];
|
|
50
|
-
if (binding.kind === "postgres") continue;
|
|
51
|
-
if (binding.kind !== "object") {
|
|
52
|
-
throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", `row binding ${id} has an unsupported kind`);
|
|
53
|
-
}
|
|
54
|
-
const provider = this.providers[binding.provider];
|
|
55
|
-
if (!provider) throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", `row binding ${id} names an unknown provider`);
|
|
56
|
-
provider.validateLocation(binding.location);
|
|
57
|
-
(0, import_row_storage_object.assertReportingRowPrefixV1)(binding.prefix);
|
|
58
|
-
if (binding.credentialRef !== void 0 && !CREDENTIAL_REF.test(binding.credentialRef)) {
|
|
59
|
-
throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", `row binding ${id} has an invalid credentialRef`);
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
this.bindings = Object.freeze(
|
|
63
|
-
Object.fromEntries(Object.entries(bindings).map(([id, binding]) => [id, Object.freeze({ ...binding })]))
|
|
64
|
-
);
|
|
65
|
-
this.assertBindingKeyTemplates();
|
|
66
|
-
this.deploymentNamespace = options.deploymentNamespace ?? "default";
|
|
67
|
-
if (!this.deploymentNamespace || this.deploymentNamespace.length > 256) {
|
|
68
|
-
throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", "deploymentNamespace must be 1-256 characters");
|
|
69
|
-
}
|
|
70
|
-
this.writeDeadline = positiveMilliseconds(options.writeDeadlineMilliseconds ?? 6e4, "writeDeadlineMilliseconds");
|
|
71
|
-
this.readDeadline = positiveMilliseconds(options.readDeadlineMilliseconds ?? 3e4, "readDeadlineMilliseconds");
|
|
72
|
-
}
|
|
73
|
-
pool;
|
|
74
|
-
options;
|
|
75
|
-
bindings;
|
|
76
|
-
providers;
|
|
77
|
-
deploymentNamespace;
|
|
78
|
-
writeDeadline;
|
|
79
|
-
readDeadline;
|
|
80
|
-
readied;
|
|
81
|
-
installation;
|
|
82
|
-
/** Deadline budget a writer must still hold on its lease before uploading. */
|
|
83
|
-
get writeDeadlineMilliseconds() {
|
|
84
|
-
return this.writeDeadline;
|
|
85
|
-
}
|
|
86
|
-
/** Every writable object binding's key template must include `{revision_key}`; throws `INVALID_INPUT`. */
|
|
87
|
-
assertBindingKeyTemplates() {
|
|
88
|
-
for (const [id, binding] of Object.entries(this.bindings)) {
|
|
89
|
-
if (binding.kind !== "object") continue;
|
|
90
|
-
try {
|
|
91
|
-
(0, import_row_storage_object.assertReportingRowKeyTemplateV1)(binding.keyTemplate ?? import_row_storage_object.REPORTING_ROW_DEFAULT_KEY_TEMPLATE, {
|
|
92
|
-
requireRevisionKey: binding.readOnly !== true
|
|
93
|
-
});
|
|
94
|
-
} catch (error) {
|
|
95
|
-
if ((0, import_row_storage_errors.isReportingRowStoreError)(error)) {
|
|
96
|
-
throw new import_row_storage_errors.ReportingRowStoreError(
|
|
97
|
-
"INVALID_INPUT",
|
|
98
|
-
`row binding ${id}: ${error.message.replace(/^Reporting row store: INVALID_INPUT: /, "")}`
|
|
99
|
-
);
|
|
100
|
-
}
|
|
101
|
-
throw error;
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
/**
|
|
106
|
-
* Refuse a configuration in which compaction could delete an object a concurrent commit is
|
|
107
|
-
* adopting. An object binding whose key template omits `{revision_key}` can map two revisions
|
|
108
|
-
* with identical chunk content to one object key, and a create-only write adopts an existing
|
|
109
|
-
* object with identical bytes. Compaction deleting that object at the same moment would leave
|
|
110
|
-
* the newer revision without its rows. Every writable binding is already required to include
|
|
111
|
-
* `{revision_key}`, so this cannot fail for a validated configuration. A `readOnly` binding is
|
|
112
|
-
* skipped: it never takes commits, so nothing can adopt through it, and every delete is
|
|
113
|
-
* reference-checked.
|
|
114
|
-
*
|
|
115
|
-
* Since the reference-checked, per-object-locked delete this is no longer the only guard: every
|
|
116
|
-
* delete path now skips an object another row set still references. It is kept as defense in
|
|
117
|
-
* depth for compaction, the one path that deletes content while the obligation stays live.
|
|
118
|
-
*/
|
|
119
|
-
assertCompactionSafeKeyTemplates() {
|
|
120
|
-
const unsafe = Object.entries(this.bindings).filter(
|
|
121
|
-
([, binding]) => binding.kind === "object" && binding.readOnly !== true && !(binding.keyTemplate ?? import_row_storage_object.REPORTING_ROW_DEFAULT_KEY_TEMPLATE).includes("{revision_key}")
|
|
122
|
-
).map(([id]) => id);
|
|
123
|
-
if (unsafe.length > 0) {
|
|
124
|
-
throw new import_row_storage_errors.ReportingRowStoreError(
|
|
125
|
-
"INVALID_INPUT",
|
|
126
|
-
`snapshot compaction requires every object row binding's keyTemplate to include {revision_key}; binding(s) ${unsafe.join(", ")} omit it, so two revisions with identical content could share an object that compaction deletes while a concurrent commit adopts it`
|
|
127
|
-
);
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
/** IDs of the object bindings this host configures, the only ones it can delete objects through. */
|
|
131
|
-
objectBindingIds() {
|
|
132
|
-
return Object.entries(this.bindings).filter(([, binding]) => binding.kind === "object").map(([id]) => id);
|
|
133
|
-
}
|
|
134
|
-
/** Probe the schema and every provider, and register (or verify) every configured binding. Idempotent. */
|
|
135
|
-
ready() {
|
|
136
|
-
this.readied ??= this.register().catch((error) => {
|
|
137
|
-
this.readied = void 0;
|
|
138
|
-
throw error;
|
|
139
|
-
});
|
|
140
|
-
return this.readied;
|
|
141
|
-
}
|
|
142
|
-
/** Encode rows, prove the chunks reproduce the committed binding, select a binding and plan objects. */
|
|
143
|
-
async prepare(input) {
|
|
144
|
-
await this.ready();
|
|
145
|
-
let encoded;
|
|
146
|
-
try {
|
|
147
|
-
encoded = (0, import_row_encoding.encodeReportingRowsV1)(input.rows);
|
|
148
|
-
} catch (error) {
|
|
149
|
-
if ((0, import_row_encoding.isReportingRowEncodingError)(error)) throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", error.message);
|
|
150
|
-
throw error;
|
|
151
|
-
}
|
|
152
|
-
const digestProfile = input.rowSetKind === "revision" ? "revision_envelope_v1" : "rows_v1";
|
|
153
|
-
const hasher = digestProfile === "revision_envelope_v1" ? (0, import_row_encoding.createReportingRevisionEnvelopeHasherV1)({
|
|
154
|
-
reporting_revision_id: input.rowSetId,
|
|
155
|
-
row_count: input.rows.length,
|
|
156
|
-
control_totals: input.controlTotals ?? []
|
|
157
|
-
}) : (0, import_row_encoding.createReportingRowsOnlyHasherV1)(input.rows.length);
|
|
158
|
-
for (const chunk of encoded.chunks) hasher.update(chunk.bytes);
|
|
159
|
-
const digest = hasher.digest();
|
|
160
|
-
if (digest.sha256 !== input.binding.sha256 || digest.byteCount !== input.binding.byteCount || input.binding.rowCount !== input.rows.length) {
|
|
161
|
-
throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", "rows do not reproduce their committed binding");
|
|
162
|
-
}
|
|
163
|
-
const writable = Object.keys(this.bindings).filter((id) => {
|
|
164
|
-
const candidate = this.bindings[id];
|
|
165
|
-
return candidate.kind !== "object" || candidate.readOnly !== true;
|
|
166
|
-
});
|
|
167
|
-
if (writable.length === 0 && !input.bindingId && !this.options.selectRowBinding) {
|
|
168
|
-
throw new import_row_storage_errors.ReportingRowStoreError(
|
|
169
|
-
"UNSAFE_BINDING",
|
|
170
|
-
"no writable row binding is configured: every binding is readOnly"
|
|
171
|
-
);
|
|
172
|
-
}
|
|
173
|
-
const bindingId = input.bindingId ? input.bindingId : this.options.selectRowBinding ? await this.options.selectRowBinding({
|
|
174
|
-
account_id: input.accountId,
|
|
175
|
-
reporting_obligation_id: input.obligationId,
|
|
176
|
-
row_set_kind: input.rowSetKind,
|
|
177
|
-
...input.finality ? { finality: input.finality } : {},
|
|
178
|
-
row_count: input.rows.length,
|
|
179
|
-
content_size_bytes: digest.byteCount
|
|
180
|
-
}) : writable.length === 1 ? writable[0] : DEFAULT_BINDING_ID;
|
|
181
|
-
const binding = this.bindings[bindingId];
|
|
182
|
-
if (!binding) throw new import_row_storage_errors.ReportingRowStoreError("STATE_UNAVAILABLE", `row binding ${bindingId} is not configured`);
|
|
183
|
-
if (binding.kind === "object" && binding.readOnly === true) {
|
|
184
|
-
throw new import_row_storage_errors.ReportingRowStoreError(
|
|
185
|
-
"UNSAFE_BINDING",
|
|
186
|
-
`row binding ${bindingId} is read-only and cannot take new row sets`
|
|
187
|
-
);
|
|
188
|
-
}
|
|
189
|
-
if (binding.kind === "postgres") return Object.freeze({ input, encoded, bindingId, binding, digestProfile });
|
|
190
|
-
const compression = binding.compression ?? "gzip";
|
|
191
|
-
const namespaceKey = this.installation.namespaceKey;
|
|
192
|
-
const objects = encoded.chunks.map((chunk) => {
|
|
193
|
-
const physicalBytes = compression === "gzip" ? (0, import_zlib.gzipSync)(chunk.bytes, { level: 6 }) : chunk.bytes;
|
|
194
|
-
return Object.freeze({
|
|
195
|
-
objectRef: (0, import_row_storage_object.renderReportingRowObjectKeyV1)({
|
|
196
|
-
template: binding.keyTemplate ?? import_row_storage_object.REPORTING_ROW_DEFAULT_KEY_TEMPLATE,
|
|
197
|
-
prefix: binding.prefix,
|
|
198
|
-
namespaceKey,
|
|
199
|
-
accountId: input.accountId,
|
|
200
|
-
rowSetId: input.rowSetId,
|
|
201
|
-
...input.periodStart ? { periodStart: input.periodStart } : {},
|
|
202
|
-
...input.sourceTimezone ? { sourceTimezone: input.sourceTimezone } : {},
|
|
203
|
-
...input.finality ? { finality: input.finality } : {},
|
|
204
|
-
contentSha256: input.binding.sha256,
|
|
205
|
-
chunkIndex: chunk.manifest.chunk_index,
|
|
206
|
-
compression
|
|
207
|
-
}),
|
|
208
|
-
physicalBytes,
|
|
209
|
-
physicalSha256: sha256Hex(physicalBytes),
|
|
210
|
-
compression
|
|
211
|
-
});
|
|
212
|
-
});
|
|
213
|
-
return Object.freeze({ input, encoded, bindingId, binding, digestProfile, objects: Object.freeze(objects) });
|
|
214
|
-
}
|
|
215
|
-
/**
|
|
216
|
-
* Upload a prepared object-kind row set, create-only. An object already at a
|
|
217
|
-
* planned key is adopted only when its bytes decode to the exact chunk;
|
|
218
|
-
* otherwise `CONTENT_CONFLICT`. `onCreated` is awaited for every object this
|
|
219
|
-
* call created, before the next upload, so the caller can record it in its
|
|
220
|
-
* write intent.
|
|
221
|
-
*/
|
|
222
|
-
async upload(prepared, intentId, onCreated, signal) {
|
|
223
|
-
const binding = prepared.binding;
|
|
224
|
-
if (binding.kind !== "object" || !prepared.objects) {
|
|
225
|
-
throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", "only object-kind row sets are uploaded");
|
|
226
|
-
}
|
|
227
|
-
const provider = this.providers[binding.provider];
|
|
228
|
-
const context = { signal: deadlineSignal(this.writeDeadline, signal) };
|
|
229
|
-
const location = providerLocation(binding);
|
|
230
|
-
const locators = [];
|
|
231
|
-
for (const [index, object] of prepared.objects.entries()) {
|
|
232
|
-
const manifest = prepared.encoded.manifests[index];
|
|
233
|
-
const put = await this.provider(
|
|
234
|
-
() => provider.putIfAbsent(
|
|
235
|
-
{
|
|
236
|
-
...location,
|
|
237
|
-
objectRef: object.objectRef,
|
|
238
|
-
bytes: object.physicalBytes,
|
|
239
|
-
contentType: object.compression === "gzip" ? "application/gzip" : "application/x-ndjson",
|
|
240
|
-
metadata: {
|
|
241
|
-
"adcp-installation": this.installation.installationId,
|
|
242
|
-
"adcp-intent": intentId
|
|
243
|
-
}
|
|
244
|
-
},
|
|
245
|
-
context
|
|
246
|
-
)
|
|
247
|
-
);
|
|
248
|
-
if (put.created) {
|
|
249
|
-
await onCreated({ objectRef: object.objectRef, nativeVersionRef: put.nativeVersionRef });
|
|
250
|
-
assertNativeVersionRef(put.nativeVersionRef);
|
|
251
|
-
locators.push({
|
|
252
|
-
objectRef: object.objectRef,
|
|
253
|
-
nativeVersionRef: put.nativeVersionRef,
|
|
254
|
-
compression: object.compression,
|
|
255
|
-
physicalSha256: object.physicalSha256,
|
|
256
|
-
sizeBytes: object.physicalBytes.byteLength,
|
|
257
|
-
created: true
|
|
258
|
-
});
|
|
259
|
-
continue;
|
|
260
|
-
}
|
|
261
|
-
assertNativeVersionRef(put.nativeVersionRef);
|
|
262
|
-
const existing = await this.provider(
|
|
263
|
-
() => provider.get(
|
|
264
|
-
{
|
|
265
|
-
...location,
|
|
266
|
-
objectRef: object.objectRef,
|
|
267
|
-
nativeVersionRef: put.nativeVersionRef,
|
|
268
|
-
maxBytes: MAX_ADOPTED_OBJECT_BYTES
|
|
269
|
-
},
|
|
270
|
-
context
|
|
271
|
-
)
|
|
272
|
-
);
|
|
273
|
-
const logical = existing ? this.inflate(Buffer.from(existing), object.compression, manifest.byte_count) : null;
|
|
274
|
-
if (!existing || !logical || sha256Hex(logical) !== manifest.sha256) {
|
|
275
|
-
throw new import_row_storage_errors.ReportingRowStoreError("CONTENT_CONFLICT", `object for chunk ${index} holds different bytes`);
|
|
276
|
-
}
|
|
277
|
-
locators.push({
|
|
278
|
-
objectRef: object.objectRef,
|
|
279
|
-
nativeVersionRef: put.nativeVersionRef,
|
|
280
|
-
compression: object.compression,
|
|
281
|
-
physicalSha256: sha256Hex(existing),
|
|
282
|
-
sizeBytes: existing.byteLength,
|
|
283
|
-
created: false
|
|
284
|
-
});
|
|
285
|
-
}
|
|
286
|
-
return locators;
|
|
287
|
-
}
|
|
288
|
-
/**
|
|
289
|
-
* Persist a prepared row set inside the caller's ledger transaction:
|
|
290
|
-
* bodies for the `postgres` kind, or the uploaded locators for the object
|
|
291
|
-
* kind. Replays of an identical row set are no-ops; a conflicting row set
|
|
292
|
-
* under the same ID is refused.
|
|
293
|
-
*/
|
|
294
|
-
async writeInTransaction(transaction, prepared, locators, sharedFromRowSetId) {
|
|
295
|
-
const { input, encoded } = prepared;
|
|
296
|
-
if (prepared.binding.kind === "object" && locators?.length !== encoded.chunks.length) {
|
|
297
|
-
throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", "object-kind row sets need one locator per chunk");
|
|
298
|
-
}
|
|
299
|
-
const inserted = await transaction.query(
|
|
300
|
-
`INSERT INTO adcp_reporting_row_sets
|
|
301
|
-
(row_set_id, row_set_kind, account_id, obligation_id, format, algorithm, digest_profile, content_sha256,
|
|
302
|
-
content_size_bytes, row_count, row_manifest_sha256, chunk_count, row_binding_id,
|
|
303
|
-
rows_shared_from_row_set_id)
|
|
304
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
|
|
305
|
-
ON CONFLICT (row_set_id) DO NOTHING`,
|
|
306
|
-
[
|
|
307
|
-
input.rowSetId,
|
|
308
|
-
input.rowSetKind,
|
|
309
|
-
input.accountId,
|
|
310
|
-
input.obligationId,
|
|
311
|
-
import_row_encoding.REPORTING_ROW_FORMAT_V1,
|
|
312
|
-
import_row_encoding.REPORTING_ROW_ALGORITHM_V1,
|
|
313
|
-
prepared.digestProfile,
|
|
314
|
-
input.binding.sha256,
|
|
315
|
-
input.binding.byteCount,
|
|
316
|
-
encoded.rowCount,
|
|
317
|
-
encoded.rowManifestSha256,
|
|
318
|
-
encoded.chunks.length,
|
|
319
|
-
prepared.bindingId,
|
|
320
|
-
sharedFromRowSetId ?? null
|
|
321
|
-
]
|
|
322
|
-
);
|
|
323
|
-
if (inserted.rowCount !== 1) {
|
|
324
|
-
const existing = await transaction.query(
|
|
325
|
-
"SELECT content_sha256, row_manifest_sha256 FROM adcp_reporting_row_sets WHERE row_set_id = $1",
|
|
326
|
-
[input.rowSetId]
|
|
327
|
-
);
|
|
328
|
-
const row = existing.rows[0];
|
|
329
|
-
if (!row || row.content_sha256 !== input.binding.sha256 || row.row_manifest_sha256 !== encoded.rowManifestSha256) {
|
|
330
|
-
throw new import_row_storage_errors.ReportingRowStoreError("CONTENT_CONFLICT", "row set identity names different content");
|
|
331
|
-
}
|
|
332
|
-
return;
|
|
333
|
-
}
|
|
334
|
-
for (const [index, chunk] of encoded.chunks.entries()) {
|
|
335
|
-
const manifest = chunk.manifest;
|
|
336
|
-
const locator = locators?.[index];
|
|
337
|
-
if (!locator) {
|
|
338
|
-
await transaction.query(
|
|
339
|
-
`INSERT INTO adcp_reporting_chunk_bodies (account_id, obligation_id, sha256, body)
|
|
340
|
-
VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING`,
|
|
341
|
-
[input.accountId, input.obligationId, manifest.sha256, chunk.bytes]
|
|
342
|
-
);
|
|
343
|
-
}
|
|
344
|
-
await transaction.query(
|
|
345
|
-
`INSERT INTO adcp_reporting_row_chunks
|
|
346
|
-
(row_set_id, chunk_index, account_id, obligation_id, first_ordinal, row_count, canonical_size_bytes, sha256,
|
|
347
|
-
segments, object_ref, native_version_ref, compression, physical_sha256, size_bytes)
|
|
348
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10, $11, $12, $13, $14)`,
|
|
349
|
-
[
|
|
350
|
-
input.rowSetId,
|
|
351
|
-
manifest.chunk_index,
|
|
352
|
-
input.accountId,
|
|
353
|
-
input.obligationId,
|
|
354
|
-
manifest.first_ordinal,
|
|
355
|
-
manifest.row_count,
|
|
356
|
-
manifest.byte_count,
|
|
357
|
-
manifest.sha256,
|
|
358
|
-
JSON.stringify(manifest.segments),
|
|
359
|
-
locator?.objectRef ?? null,
|
|
360
|
-
locator?.nativeVersionRef ?? null,
|
|
361
|
-
locator?.compression ?? "none",
|
|
362
|
-
locator?.physicalSha256 ?? null,
|
|
363
|
-
locator?.sizeBytes ?? null
|
|
364
|
-
]
|
|
365
|
-
);
|
|
366
|
-
}
|
|
367
|
-
}
|
|
368
|
-
/**
|
|
369
|
-
* Load a row set and verify its chunk manifest against the committed digest.
|
|
370
|
-
* Returns `null` when no row set exists (legacy inline rows) or it belongs
|
|
371
|
-
* to another account. Throws `ROWS_EXPIRED` / `ROWS_UNAVAILABLE` when the
|
|
372
|
-
* rows were pruned or lost.
|
|
373
|
-
*/
|
|
374
|
-
async find(rowSetId, accountId, queryable = this.pool) {
|
|
375
|
-
const found = await queryable.query(
|
|
376
|
-
`SELECT row_set.row_set_id, row_set.row_set_kind, row_set.account_id, row_set.obligation_id,
|
|
377
|
-
row_set.digest_profile, row_set.content_sha256, row_set.content_size_bytes, row_set.row_count,
|
|
378
|
-
row_set.row_manifest_sha256, row_set.chunk_count, row_set.row_binding_id,
|
|
379
|
-
row_set.row_location_version, row_set.rows_state, binding.kind, binding.identity_sha256
|
|
380
|
-
FROM adcp_reporting_row_sets row_set
|
|
381
|
-
JOIN adcp_reporting_row_bindings binding ON binding.row_binding_id = row_set.row_binding_id
|
|
382
|
-
WHERE row_set.row_set_id = $1 AND row_set.account_id = $2`,
|
|
383
|
-
[rowSetId, accountId]
|
|
384
|
-
);
|
|
385
|
-
const record = found.rows[0];
|
|
386
|
-
if (!record) return null;
|
|
387
|
-
if (record.rows_state === "pruning" || record.rows_state === "pruned") {
|
|
388
|
-
throw new import_row_storage_errors.ReportingRowStoreError("ROWS_EXPIRED");
|
|
389
|
-
}
|
|
390
|
-
if (record.rows_state === "unavailable") throw new import_row_storage_errors.ReportingRowStoreError("ROWS_UNAVAILABLE");
|
|
391
|
-
const chunks = await queryable.query(
|
|
392
|
-
`SELECT chunk_index, first_ordinal, row_count, canonical_size_bytes, sha256, segments,
|
|
393
|
-
object_ref, native_version_ref, compression, physical_sha256, size_bytes
|
|
394
|
-
FROM adcp_reporting_row_chunks WHERE row_set_id = $1 ORDER BY chunk_index`,
|
|
395
|
-
[rowSetId]
|
|
396
|
-
);
|
|
397
|
-
const manifests = chunks.rows.map((chunk) => ({
|
|
398
|
-
chunk_index: chunk.chunk_index,
|
|
399
|
-
first_ordinal: Number(chunk.first_ordinal),
|
|
400
|
-
row_count: chunk.row_count,
|
|
401
|
-
byte_count: Number(chunk.canonical_size_bytes),
|
|
402
|
-
sha256: chunk.sha256,
|
|
403
|
-
segments: chunk.segments.map((segment) => ({
|
|
404
|
-
first_ordinal: Number(segment.first_ordinal),
|
|
405
|
-
row_count: Number(segment.row_count),
|
|
406
|
-
byte_offset: Number(segment.byte_offset),
|
|
407
|
-
byte_count: Number(segment.byte_count),
|
|
408
|
-
sha256: segment.sha256
|
|
409
|
-
}))
|
|
410
|
-
}));
|
|
411
|
-
const rowCount = Number(record.row_count);
|
|
412
|
-
try {
|
|
413
|
-
(0, import_row_encoding.verifyReportingRowManifestsV1)(manifests, { rowCount, rowManifestSha256: record.row_manifest_sha256 });
|
|
414
|
-
} catch (error) {
|
|
415
|
-
if ((0, import_row_encoding.isReportingRowEncodingError)(error)) throw new import_row_storage_errors.ReportingRowStoreError("ROWS_INTEGRITY_FAILED", error.message);
|
|
416
|
-
throw error;
|
|
417
|
-
}
|
|
418
|
-
if (manifests.length !== record.chunk_count) {
|
|
419
|
-
throw new import_row_storage_errors.ReportingRowStoreError("ROWS_INTEGRITY_FAILED", "chunk count does not match the row set");
|
|
420
|
-
}
|
|
421
|
-
const locators = chunks.rows.map((chunk) => {
|
|
422
|
-
const hasLocation = chunk.object_ref !== null;
|
|
423
|
-
if (hasLocation !== (record.kind === "object")) {
|
|
424
|
-
throw new import_row_storage_errors.ReportingRowStoreError("ROWS_INTEGRITY_FAILED", "chunk location does not match its binding kind");
|
|
425
|
-
}
|
|
426
|
-
return hasLocation ? Object.freeze({
|
|
427
|
-
objectRef: chunk.object_ref,
|
|
428
|
-
nativeVersionRef: chunk.native_version_ref,
|
|
429
|
-
compression: chunk.compression,
|
|
430
|
-
physicalSha256: chunk.physical_sha256,
|
|
431
|
-
sizeBytes: Number(chunk.size_bytes)
|
|
432
|
-
}) : void 0;
|
|
433
|
-
});
|
|
434
|
-
return Object.freeze({
|
|
435
|
-
rowSetId: record.row_set_id,
|
|
436
|
-
rowSetKind: record.row_set_kind,
|
|
437
|
-
accountId: record.account_id,
|
|
438
|
-
obligationId: record.obligation_id,
|
|
439
|
-
digestProfile: record.digest_profile,
|
|
440
|
-
contentSha256: record.content_sha256,
|
|
441
|
-
contentSizeBytes: Number(record.content_size_bytes),
|
|
442
|
-
rowCount,
|
|
443
|
-
rowManifestSha256: record.row_manifest_sha256,
|
|
444
|
-
bindingId: record.row_binding_id,
|
|
445
|
-
bindingIdentitySha256: record.identity_sha256,
|
|
446
|
-
kind: record.kind,
|
|
447
|
-
manifests: Object.freeze(manifests),
|
|
448
|
-
locators: Object.freeze(locators)
|
|
449
|
-
});
|
|
450
|
-
}
|
|
451
|
-
/** Verified rows `[offset, offset + limit)` in ordinal order. */
|
|
452
|
-
async readPage(handle, offset, limit, signal) {
|
|
453
|
-
if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1) {
|
|
454
|
-
throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", "row page bounds are invalid");
|
|
455
|
-
}
|
|
456
|
-
const end = Math.min(handle.rowCount, offset + limit);
|
|
457
|
-
const rows = [];
|
|
458
|
-
for (const manifest of handle.manifests) {
|
|
459
|
-
if (offset >= end) break;
|
|
460
|
-
const chunkEnd = manifest.first_ordinal + manifest.row_count;
|
|
461
|
-
if (chunkEnd <= offset || manifest.first_ordinal >= end) continue;
|
|
462
|
-
signal?.throwIfAborted();
|
|
463
|
-
const from = manifest.segments.findIndex(
|
|
464
|
-
(segment) => segment.first_ordinal + segment.row_count > Math.max(offset, manifest.first_ordinal)
|
|
465
|
-
);
|
|
466
|
-
let to = from;
|
|
467
|
-
while (to + 1 < manifest.segments.length && manifest.segments[to + 1].first_ordinal < end) to += 1;
|
|
468
|
-
const firstSegment = manifest.segments[from];
|
|
469
|
-
const lastSegment = manifest.segments[to];
|
|
470
|
-
const bytes = await this.readChunkRange(
|
|
471
|
-
handle,
|
|
472
|
-
manifest,
|
|
473
|
-
firstSegment.byte_offset,
|
|
474
|
-
lastSegment.byte_offset + lastSegment.byte_count - firstSegment.byte_offset,
|
|
475
|
-
signal
|
|
476
|
-
);
|
|
477
|
-
const decoded = this.decode(() => (0, import_row_encoding.decodeVerifiedReportingRowSegmentsV1)(manifest, [from, to], bytes));
|
|
478
|
-
const startIndex = Math.max(offset, firstSegment.first_ordinal) - firstSegment.first_ordinal;
|
|
479
|
-
const endIndex = Math.min(end, lastSegment.first_ordinal + lastSegment.row_count) - firstSegment.first_ordinal;
|
|
480
|
-
rows.push(...decoded.slice(startIndex, endIndex));
|
|
481
|
-
}
|
|
482
|
-
return rows;
|
|
483
|
-
}
|
|
484
|
-
/**
|
|
485
|
-
* Every row, verified chunk by chunk and against the committed binding.
|
|
486
|
-
* `controlTotals` are required for revision row sets.
|
|
487
|
-
*/
|
|
488
|
-
async readAll(handle, controlTotals, options = {}) {
|
|
489
|
-
if (options.maxBytes !== void 0 && handle.contentSizeBytes > options.maxBytes) {
|
|
490
|
-
throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", "row set exceeds the hydration byte limit");
|
|
491
|
-
}
|
|
492
|
-
const hasher = handle.digestProfile === "revision_envelope_v1" ? (0, import_row_encoding.createReportingRevisionEnvelopeHasherV1)({
|
|
493
|
-
reporting_revision_id: handle.rowSetId,
|
|
494
|
-
row_count: handle.rowCount,
|
|
495
|
-
control_totals: controlTotals ?? []
|
|
496
|
-
}) : (0, import_row_encoding.createReportingRowsOnlyHasherV1)(handle.rowCount);
|
|
497
|
-
const rows = [];
|
|
498
|
-
for (const manifest of handle.manifests) {
|
|
499
|
-
options.signal?.throwIfAborted();
|
|
500
|
-
const bytes = await this.readChunkRange(handle, manifest, 0, manifest.byte_count, options.signal);
|
|
501
|
-
this.decode(() => (0, import_row_encoding.verifyReportingRowChunkV1)(manifest, bytes));
|
|
502
|
-
hasher.update(bytes);
|
|
503
|
-
rows.push(
|
|
504
|
-
...this.decode(() => (0, import_row_encoding.decodeVerifiedReportingRowSegmentsV1)(manifest, [0, manifest.segments.length - 1], bytes))
|
|
505
|
-
);
|
|
506
|
-
}
|
|
507
|
-
const digest = this.decode(() => hasher.digest());
|
|
508
|
-
if (digest.sha256 !== handle.contentSha256 || digest.byteCount !== handle.contentSizeBytes) {
|
|
509
|
-
throw new import_row_storage_errors.ReportingRowStoreError("ROWS_INTEGRITY_FAILED", "rows do not reproduce the committed binding");
|
|
510
|
-
}
|
|
511
|
-
return rows;
|
|
512
|
-
}
|
|
513
|
-
/**
|
|
514
|
-
* Delete exact object versions through the configured binding's provider.
|
|
515
|
-
* `absent` counts as success. Only object-kind bindings are accepted.
|
|
516
|
-
*/
|
|
517
|
-
async deleteObjects(bindingId, objects, signal) {
|
|
518
|
-
await this.ready();
|
|
519
|
-
const binding = this.bindings[bindingId];
|
|
520
|
-
if (!binding || binding.kind !== "object") {
|
|
521
|
-
throw new import_row_storage_errors.ReportingRowStoreError("STATE_UNAVAILABLE", `row binding ${bindingId} is not configured for objects`);
|
|
522
|
-
}
|
|
523
|
-
const provider = this.providers[binding.provider];
|
|
524
|
-
const ownedPrefix = `${binding.prefix}/${this.installation.namespaceKey}/`;
|
|
525
|
-
const context = { signal: deadlineSignal(this.writeDeadline, signal) };
|
|
526
|
-
for (const object of objects) {
|
|
527
|
-
if (!object.objectRef.startsWith(ownedPrefix)) {
|
|
528
|
-
throw new import_row_storage_errors.ReportingRowStoreError("UNSAFE_BINDING", "refusing to delete an object outside the owned prefix");
|
|
529
|
-
}
|
|
530
|
-
await this.provider(
|
|
531
|
-
() => provider.delete(
|
|
532
|
-
{ ...providerLocation(binding), objectRef: object.objectRef, nativeVersionRef: object.nativeVersionRef },
|
|
533
|
-
context
|
|
534
|
-
)
|
|
535
|
-
);
|
|
536
|
-
}
|
|
537
|
-
}
|
|
538
|
-
/**
|
|
539
|
-
* Prove each object is still present at its recorded version with the recorded size.
|
|
540
|
-
* A commit that adopted an object another row set created calls this under the object's
|
|
541
|
-
* lock, so a concurrent delete either finished first (this throws, the commit fails and
|
|
542
|
-
* the writer re-uploads) or sees the commit's chunk reference and skips the delete.
|
|
543
|
-
*
|
|
544
|
-
* Native versions are write-unique (S3 VersionId, Azure versionId, GCS generation, filesystem
|
|
545
|
-
* `ino:birthtime:mtime:size`), so a version-pinned metadata check, "this exact version exists at the
|
|
546
|
-
* recorded size", proves the object unchanged without downloading it while the commit holds its
|
|
547
|
-
* locks. Bytes were verified when the object was uploaded or adopted. A custom provider without
|
|
548
|
-
* `stat` falls back to reading the whole object and comparing its physical digest.
|
|
549
|
-
*/
|
|
550
|
-
async assertObjectsPresent(bindingId, objects, signal) {
|
|
551
|
-
await this.ready();
|
|
552
|
-
const binding = this.bindings[bindingId];
|
|
553
|
-
if (!binding || binding.kind !== "object") {
|
|
554
|
-
throw new import_row_storage_errors.ReportingRowStoreError("STATE_UNAVAILABLE", `row binding ${bindingId} is not configured for objects`);
|
|
555
|
-
}
|
|
556
|
-
const provider = this.providers[binding.provider];
|
|
557
|
-
const context = { signal: deadlineSignal(this.writeDeadline, signal) };
|
|
558
|
-
const location = providerLocation(binding);
|
|
559
|
-
const changed = () => new import_row_storage_errors.ReportingRowStoreError(
|
|
560
|
-
"STATE_UNAVAILABLE",
|
|
561
|
-
"an adopted row object was deleted or replaced before its commit; retry the upload"
|
|
562
|
-
);
|
|
563
|
-
for (const object of objects) {
|
|
564
|
-
const identity = { ...location, objectRef: object.objectRef, nativeVersionRef: object.nativeVersionRef };
|
|
565
|
-
if (provider.stat) {
|
|
566
|
-
const stat = await this.provider(() => provider.stat(identity, context));
|
|
567
|
-
if (!stat || stat.nativeVersionRef !== object.nativeVersionRef || !Number.isSafeInteger(stat.sizeBytes) || stat.sizeBytes !== object.sizeBytes) {
|
|
568
|
-
throw changed();
|
|
569
|
-
}
|
|
570
|
-
continue;
|
|
571
|
-
}
|
|
572
|
-
const bytes = await this.provider(
|
|
573
|
-
() => provider.get({ ...identity, maxBytes: MAX_ADOPTED_OBJECT_BYTES }, context)
|
|
574
|
-
);
|
|
575
|
-
if (!bytes || sha256Hex(bytes) !== object.physicalSha256) throw changed();
|
|
576
|
-
}
|
|
577
|
-
}
|
|
578
|
-
/**
|
|
579
|
-
* A live row set in the same obligation and binding whose rows are
|
|
580
|
-
* byte-identical (same manifest digest), with its chunk locations. Sharing
|
|
581
|
-
* never crosses obligations, so it never crosses consumers.
|
|
582
|
-
*/
|
|
583
|
-
async findShareable(queryable, prepared) {
|
|
584
|
-
if (prepared.encoded.chunks.length === 0) return void 0;
|
|
585
|
-
const found = await queryable.query(
|
|
586
|
-
`SELECT row_set_id FROM adcp_reporting_row_sets
|
|
587
|
-
WHERE obligation_id = $1 AND account_id = $2 AND row_binding_id = $3 AND row_manifest_sha256 = $4
|
|
588
|
-
AND rows_state = 'live' AND row_set_id <> $5
|
|
589
|
-
ORDER BY recorded_at DESC, row_set_id LIMIT 1
|
|
590
|
-
FOR SHARE`,
|
|
591
|
-
[
|
|
592
|
-
prepared.input.obligationId,
|
|
593
|
-
prepared.input.accountId,
|
|
594
|
-
prepared.bindingId,
|
|
595
|
-
prepared.encoded.rowManifestSha256,
|
|
596
|
-
prepared.input.rowSetId
|
|
597
|
-
]
|
|
598
|
-
);
|
|
599
|
-
const rowSetId = found.rows[0]?.row_set_id;
|
|
600
|
-
if (!rowSetId) return void 0;
|
|
601
|
-
const chunks = await queryable.query(
|
|
602
|
-
`SELECT object_ref, native_version_ref, compression, physical_sha256, size_bytes
|
|
603
|
-
FROM adcp_reporting_row_chunks WHERE row_set_id = $1 ORDER BY chunk_index`,
|
|
604
|
-
[rowSetId]
|
|
605
|
-
);
|
|
606
|
-
if (chunks.rows.length !== prepared.encoded.chunks.length) return void 0;
|
|
607
|
-
if (prepared.binding.kind === "postgres") return { rowSetId, locators: [] };
|
|
608
|
-
if (chunks.rows.some((chunk) => chunk.object_ref === null)) return void 0;
|
|
609
|
-
return {
|
|
610
|
-
rowSetId,
|
|
611
|
-
locators: chunks.rows.map((chunk) => ({
|
|
612
|
-
objectRef: chunk.object_ref,
|
|
613
|
-
nativeVersionRef: chunk.native_version_ref,
|
|
614
|
-
compression: chunk.compression,
|
|
615
|
-
physicalSha256: chunk.physical_sha256,
|
|
616
|
-
sizeBytes: Number(chunk.size_bytes),
|
|
617
|
-
created: false
|
|
618
|
-
}))
|
|
619
|
-
};
|
|
620
|
-
}
|
|
621
|
-
/** Installation identity and namespace key; available after `ready()`. */
|
|
622
|
-
get installationIdentity() {
|
|
623
|
-
if (!this.installation) throw new import_row_storage_errors.ReportingRowStoreError("STATE_UNAVAILABLE", "row storage is not ready");
|
|
624
|
-
return this.installation;
|
|
625
|
-
}
|
|
626
|
-
async readChunkRange(handle, manifest, byteOffset, byteCount, signal) {
|
|
627
|
-
if (handle.kind === "postgres") {
|
|
628
|
-
const result = await this.pool.query(
|
|
629
|
-
`SELECT substring(body FROM $4::integer + 1 FOR $5::integer) AS bytes
|
|
630
|
-
FROM adcp_reporting_chunk_bodies
|
|
631
|
-
WHERE account_id = $1 AND obligation_id = $2 AND sha256 = $3`,
|
|
632
|
-
[handle.accountId, handle.obligationId, manifest.sha256, byteOffset, byteCount]
|
|
633
|
-
);
|
|
634
|
-
const bytes = result.rows[0]?.bytes;
|
|
635
|
-
if (!bytes) throw new import_row_storage_errors.ReportingRowStoreError("ROWS_UNAVAILABLE", `chunk ${manifest.chunk_index} body is missing`);
|
|
636
|
-
return bytes;
|
|
637
|
-
}
|
|
638
|
-
const binding = this.bindings[handle.bindingId];
|
|
639
|
-
if (!binding || binding.kind !== "object" || bindingIdentitySha256(binding) !== handle.bindingIdentitySha256) {
|
|
640
|
-
throw new import_row_storage_errors.ReportingRowStoreError("STATE_UNAVAILABLE", `row binding ${handle.bindingId} is not configured`);
|
|
641
|
-
}
|
|
642
|
-
const locator = handle.locators[manifest.chunk_index];
|
|
643
|
-
const provider = this.providers[binding.provider];
|
|
644
|
-
const context = { signal: deadlineSignal(this.readDeadline, signal) };
|
|
645
|
-
const location = providerLocation(binding);
|
|
646
|
-
if (locator.compression === "none" && provider.rangedReads) {
|
|
647
|
-
const bytes = await this.provider(
|
|
648
|
-
() => provider.get(
|
|
649
|
-
{
|
|
650
|
-
...location,
|
|
651
|
-
objectRef: locator.objectRef,
|
|
652
|
-
nativeVersionRef: locator.nativeVersionRef,
|
|
653
|
-
range: { offset: byteOffset, length: byteCount },
|
|
654
|
-
maxBytes: byteCount
|
|
655
|
-
},
|
|
656
|
-
context
|
|
657
|
-
)
|
|
658
|
-
);
|
|
659
|
-
if (!bytes)
|
|
660
|
-
throw new import_row_storage_errors.ReportingRowStoreError("ROWS_UNAVAILABLE", `chunk ${manifest.chunk_index} object is missing`);
|
|
661
|
-
if (bytes.byteLength !== byteCount) {
|
|
662
|
-
throw new import_row_storage_errors.ReportingRowStoreError("ROWS_INTEGRITY_FAILED", `chunk ${manifest.chunk_index} range is truncated`);
|
|
663
|
-
}
|
|
664
|
-
return Buffer.from(bytes);
|
|
665
|
-
}
|
|
666
|
-
const stored = await this.provider(
|
|
667
|
-
() => provider.get(
|
|
668
|
-
{
|
|
669
|
-
...location,
|
|
670
|
-
objectRef: locator.objectRef,
|
|
671
|
-
nativeVersionRef: locator.nativeVersionRef,
|
|
672
|
-
maxBytes: locator.sizeBytes
|
|
673
|
-
},
|
|
674
|
-
context
|
|
675
|
-
)
|
|
676
|
-
);
|
|
677
|
-
if (!stored)
|
|
678
|
-
throw new import_row_storage_errors.ReportingRowStoreError("ROWS_UNAVAILABLE", `chunk ${manifest.chunk_index} object is missing`);
|
|
679
|
-
const physical = Buffer.from(stored);
|
|
680
|
-
if (physical.byteLength !== locator.sizeBytes || sha256Hex(physical) !== locator.physicalSha256) {
|
|
681
|
-
throw new import_row_storage_errors.ReportingRowStoreError("ROWS_INTEGRITY_FAILED", `chunk ${manifest.chunk_index} stored bytes differ`);
|
|
682
|
-
}
|
|
683
|
-
const logical = this.inflate(physical, locator.compression, manifest.byte_count);
|
|
684
|
-
if (!logical) {
|
|
685
|
-
throw new import_row_storage_errors.ReportingRowStoreError("ROWS_INTEGRITY_FAILED", `chunk ${manifest.chunk_index} does not decompress`);
|
|
686
|
-
}
|
|
687
|
-
return logical.subarray(byteOffset, byteOffset + byteCount);
|
|
688
|
-
}
|
|
689
|
-
inflate(physical, compression, byteCount) {
|
|
690
|
-
if (compression === "none") return physical.byteLength === byteCount ? physical : null;
|
|
691
|
-
try {
|
|
692
|
-
const logical = (0, import_zlib.gunzipSync)(physical, { maxOutputLength: byteCount });
|
|
693
|
-
return logical.byteLength === byteCount ? logical : null;
|
|
694
|
-
} catch {
|
|
695
|
-
return null;
|
|
696
|
-
}
|
|
697
|
-
}
|
|
698
|
-
decode(operation) {
|
|
699
|
-
try {
|
|
700
|
-
return operation();
|
|
701
|
-
} catch (error) {
|
|
702
|
-
if ((0, import_row_encoding.isReportingRowEncodingError)(error)) throw new import_row_storage_errors.ReportingRowStoreError("ROWS_INTEGRITY_FAILED", error.message);
|
|
703
|
-
throw error;
|
|
704
|
-
}
|
|
705
|
-
}
|
|
706
|
-
/** Collapse provider failures to stable, secret-free codes. */
|
|
707
|
-
async provider(operation) {
|
|
708
|
-
try {
|
|
709
|
-
return await operation();
|
|
710
|
-
} catch (error) {
|
|
711
|
-
if ((0, import_row_storage_errors.isReportingRowStoreError)(error)) throw error;
|
|
712
|
-
if (error instanceof Error && (error.name === "TimeoutError" || error.name === "AbortError")) {
|
|
713
|
-
throw new import_row_storage_errors.ReportingRowStoreError(error.name === "TimeoutError" ? "DEADLINE_EXCEEDED" : "ABORTED");
|
|
714
|
-
}
|
|
715
|
-
throw new import_row_storage_errors.ReportingRowStoreError("PROVIDER_UNAVAILABLE");
|
|
716
|
-
}
|
|
717
|
-
}
|
|
718
|
-
async register() {
|
|
719
|
-
this.assertBindingKeyTemplates();
|
|
720
|
-
const { installationId } = await (0, import_row_storage_migration.probeReportingRowStorageSchemaV1)(this.pool);
|
|
721
|
-
const namespaceKey = sha256Hex((0, import_jcs.canonicalize)(["adcp.rows.v1", this.deploymentNamespace, installationId]));
|
|
722
|
-
this.installation = { installationId, namespaceKey };
|
|
723
|
-
for (const [bindingId, binding] of Object.entries(this.bindings)) {
|
|
724
|
-
if (binding.kind === "object") {
|
|
725
|
-
const provider = this.providers[binding.provider];
|
|
726
|
-
await this.provider(
|
|
727
|
-
() => provider.probe(
|
|
728
|
-
{ ...providerLocation(binding), prefix: binding.prefix },
|
|
729
|
-
{ signal: AbortSignal.timeout(this.readDeadline) }
|
|
730
|
-
)
|
|
731
|
-
);
|
|
732
|
-
}
|
|
733
|
-
const identityConfig = bindingIdentityConfig(binding);
|
|
734
|
-
const identitySha256 = bindingIdentitySha256(binding);
|
|
735
|
-
const operationalConfig = binding.kind === "object" ? {
|
|
736
|
-
compression: binding.compression ?? "gzip",
|
|
737
|
-
...binding.credentialRef ? { credentialRef: binding.credentialRef } : {}
|
|
738
|
-
} : {};
|
|
739
|
-
await this.pool.query(
|
|
740
|
-
`INSERT INTO adcp_reporting_row_bindings
|
|
741
|
-
(row_binding_id, kind, provider, identity_config, identity_sha256, operational_config, namespace_key)
|
|
742
|
-
VALUES ($1, $2, $3, $4::jsonb, $5, $6::jsonb, $7)
|
|
743
|
-
ON CONFLICT (row_binding_id) DO NOTHING`,
|
|
744
|
-
[
|
|
745
|
-
bindingId,
|
|
746
|
-
binding.kind,
|
|
747
|
-
binding.kind === "object" ? binding.provider : "postgres",
|
|
748
|
-
JSON.stringify(identityConfig),
|
|
749
|
-
identitySha256,
|
|
750
|
-
JSON.stringify(operationalConfig),
|
|
751
|
-
namespaceKey
|
|
752
|
-
]
|
|
753
|
-
);
|
|
754
|
-
const stored = await this.pool.query(
|
|
755
|
-
"SELECT identity_sha256, namespace_key, state FROM adcp_reporting_row_bindings WHERE row_binding_id = $1",
|
|
756
|
-
[bindingId]
|
|
757
|
-
);
|
|
758
|
-
const row = stored.rows[0];
|
|
759
|
-
if (!row || row.identity_sha256 !== identitySha256) {
|
|
760
|
-
throw new import_row_storage_errors.ReportingRowStoreError(
|
|
761
|
-
"UNSAFE_BINDING",
|
|
762
|
-
`row binding ${bindingId} identity differs from its definition`
|
|
763
|
-
);
|
|
764
|
-
}
|
|
765
|
-
if (row.namespace_key !== namespaceKey) {
|
|
766
|
-
throw new import_row_storage_errors.ReportingRowStoreError("UNSAFE_BINDING", `row binding ${bindingId} belongs to another installation`);
|
|
767
|
-
}
|
|
768
|
-
}
|
|
769
|
-
}
|
|
770
|
-
}
|
|
771
|
-
function bindingIdentityConfig(binding) {
|
|
772
|
-
return binding.kind === "object" ? {
|
|
773
|
-
location: { ...binding.location },
|
|
774
|
-
prefix: binding.prefix,
|
|
775
|
-
keyTemplate: binding.keyTemplate ?? import_row_storage_object.REPORTING_ROW_DEFAULT_KEY_TEMPLATE
|
|
776
|
-
} : {};
|
|
777
|
-
}
|
|
778
|
-
function bindingIdentitySha256(binding) {
|
|
779
|
-
const provider = binding.kind === "object" ? binding.provider : binding.kind;
|
|
780
|
-
return sha256Hex((0, import_jcs.canonicalize)([binding.kind, provider, bindingIdentityConfig(binding)]));
|
|
781
|
-
}
|
|
782
|
-
function providerLocation(binding) {
|
|
783
|
-
return { location: binding.location, ...binding.credentialRef ? { credentialRef: binding.credentialRef } : {} };
|
|
784
|
-
}
|
|
785
|
-
function deadlineSignal(milliseconds, signal) {
|
|
786
|
-
const timeout = AbortSignal.timeout(milliseconds);
|
|
787
|
-
return signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
788
|
-
}
|
|
789
|
-
function assertNativeVersionRef(value) {
|
|
790
|
-
if (typeof value !== "string" || value.length < 1 || value.length > import_row_storage_object.REPORTING_ROW_NATIVE_VERSION_REF_MAX_LENGTH) {
|
|
791
|
-
throw new import_row_storage_errors.ReportingRowStoreError("PROVIDER_UNAVAILABLE", "provider returned an invalid native version reference");
|
|
792
|
-
}
|
|
793
|
-
}
|
|
794
|
-
function positiveMilliseconds(value, name) {
|
|
795
|
-
if (!Number.isSafeInteger(value) || value < 1e3 || value > 6e5) {
|
|
796
|
-
throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", `${name} must be between 1000 and 600000`);
|
|
797
|
-
}
|
|
798
|
-
return value;
|
|
799
|
-
}
|
|
800
|
-
function sha256Hex(value) {
|
|
801
|
-
return (0, import_crypto.createHash)("sha256").update(value).digest("hex");
|
|
802
|
-
}
|
|
803
|
-
function reportingRowWriteIntentIdV1() {
|
|
804
|
-
return (0, import_crypto.randomUUID)();
|
|
805
|
-
}
|
|
806
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
807
|
-
0 && (module.exports = {
|
|
808
|
-
ReportingRowStorageV1,
|
|
809
|
-
ReportingRowStoreError,
|
|
810
|
-
isReportingRowStoreError,
|
|
811
|
-
reportingRowWriteIntentIdV1
|
|
812
|
-
});
|