@carbonenginejs/runtime-resource 0.18.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/format/carbonEffect/CjsCarbonEffectWriter.js +16 -4
- package/dist/format/carbonEffect/CjsCarbonEffectWriter.js.map +1 -1
- package/dist/format/carbonEffect/buildCarbonEffectContainer.js +11 -1
- package/dist/format/carbonEffect/buildCarbonEffectContainer.js.map +1 -1
- package/dist/format/carbonEffect/carbonEffectBackendBlock.js +36 -0
- package/dist/format/carbonEffect/carbonEffectBackendBlock.js.map +1 -1
- package/dist/format/carbonEffect/carbonEffectRecords.js +45 -1
- package/dist/format/carbonEffect/carbonEffectRecords.js.map +1 -1
- package/dist/format/carbonEffect/carbonEffectResourceTransform.js +120 -60
- package/dist/format/carbonEffect/carbonEffectResourceTransform.js.map +1 -1
- package/dist/format/index.js +1 -1
- package/dist/formats/dds/core/helpers.js +35 -6
- package/dist/formats/dds/core/helpers.js.map +1 -1
- package/dist/formats/hlsl/core/carbonDescriptionToRuntime.js +53 -1
- package/dist/formats/hlsl/core/carbonDescriptionToRuntime.js.map +1 -1
- package/dist/formats/hlsl/core/tr2/shader/HlslEffectBindingManifest.js +10 -1
- package/dist/formats/hlsl/core/tr2/shader/HlslEffectBindingManifest.js.map +1 -1
- package/dist/formats/index.js +3 -0
- package/dist/formats/index.js.map +1 -1
- package/dist/formats/schemabound/CjsSchemaBoundFormat.js +236 -0
- package/dist/formats/schemabound/CjsSchemaBoundFormat.js.map +1 -0
- package/dist/formats/schemabound/core/schemaBoundErrors.js +9 -0
- package/dist/formats/schemabound/core/schemaBoundErrors.js.map +1 -0
- package/dist/formats/schemabound/core/schemaBoundValues.js +307 -0
- package/dist/formats/schemabound/core/schemaBoundValues.js.map +1 -0
- package/dist/formats/schemabound/index.js +2 -0
- package/dist/formats/schemabound/index.js.map +1 -0
- package/dist/formats/sqlite/CjsSqliteFormat.js +244 -0
- package/dist/formats/sqlite/CjsSqliteFormat.js.map +1 -0
- package/dist/formats/sqlite/core/sqlitePages.js +159 -0
- package/dist/formats/sqlite/core/sqlitePages.js.map +1 -0
- package/dist/formats/sqlite/core/sqliteRecords.js +174 -0
- package/dist/formats/sqlite/core/sqliteRecords.js.map +1 -0
- package/dist/formats/sqlite/core/sqliteSchema.js +100 -0
- package/dist/formats/sqlite/core/sqliteSchema.js.map +1 -0
- package/dist/formats/sqlite/index.js +2 -0
- package/dist/formats/sqlite/index.js.map +1 -0
- package/dist/formats/static/CjsStaticFormat.js +220 -0
- package/dist/formats/static/CjsStaticFormat.js.map +1 -0
- package/dist/formats/static/index.js +2 -0
- package/dist/formats/static/index.js.map +1 -0
- package/dist/formats/webgl/core/buildGlslEffectContainer.js +3 -1
- package/dist/formats/webgl/core/buildGlslEffectContainer.js.map +1 -1
- package/dist/formats/webgl/core/effectPackage.js +11 -2
- package/dist/formats/webgl/core/effectPackage.js.map +1 -1
- package/dist/formats/webgl/core/glsl/DxbcGlslEmitter.js +10 -2
- package/dist/formats/webgl/core/glsl/DxbcGlslEmitter.js.map +1 -1
- package/dist/formats/webgpu/core/buildCarbonEffectContainer.js +3 -1
- package/dist/formats/webgpu/core/buildCarbonEffectContainer.js.map +1 -1
- package/dist/formats/webgpu/core/carbonWebgpu/containerViews.js +179 -131
- package/dist/formats/webgpu/core/carbonWebgpu/containerViews.js.map +1 -1
- package/dist/formats/webgpu/core/effectBackendBodySet.js +20 -20
- package/dist/formats/webgpu/core/effectBackendBodySet.js.map +1 -1
- package/dist/formats/webgpu/core/packageEffect.js +14 -1
- package/dist/formats/webgpu/core/packageEffect.js.map +1 -1
- package/dist/formats/webgpu/core/wgsl/buildResourceTransformPlan.js +9 -1
- package/dist/formats/webgpu/core/wgsl/buildResourceTransformPlan.js.map +1 -1
- package/dist/resource/shader/reflection/Tr2EffectStageInput.js +14 -0
- package/dist/resource/shader/reflection/Tr2EffectStageInput.js.map +1 -1
- package/docs/concepts/shader-resource-model.md +34 -1
- package/docs/formats/README.md +21 -0
- package/docs/formats/carbon-effect-container.md +54 -27
- package/docs/formats/schemabound.md +112 -0
- package/docs/formats/static.md +119 -0
- package/docs/formats/webgl/README.md +5 -3
- package/docs/formats/webgpu/README.md +2 -2
- package/docs/formats/webgpu/architecture.md +2 -2
- package/docs/formats/webgpu/formats/carbon-webgpu.md +13 -2
- package/docs/formats/webgpu/guides/effect-packaging.md +1 -2
- package/docs/formats/webgpu/reference/wgsl-compatibility.md +2 -2
- package/docs/reference/classes/formats.md +33 -0
- package/package.json +4 -1
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
import { ReadColumns } from './core/sqliteSchema.js';
|
|
2
|
+
import { ReadHeader, TableRows } from './core/sqlitePages.js';
|
|
3
|
+
import { SqliteError, ReadRecord } from './core/sqliteRecords.js';
|
|
4
|
+
|
|
5
|
+
const OUTPUT_JSON = "json";
|
|
6
|
+
const OUTPUT_PAYLOAD = "payload";
|
|
7
|
+
const OUTPUT_RAW = "raw";
|
|
8
|
+
const MASTER_ROOT_PAGE = 1;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Reads a SQLite 3 container as data.
|
|
12
|
+
*
|
|
13
|
+
* **This is a reader, not a database.** It accepts no SQL and runs no queries.
|
|
14
|
+
* It returns the tables a container holds and every row of them, because that
|
|
15
|
+
* is the whole of what this organization does with a SQLite file: the client's
|
|
16
|
+
* `.static` containers and the prepared static data export are both opened,
|
|
17
|
+
* scanned once, and closed.
|
|
18
|
+
*
|
|
19
|
+
* That scope is what makes it small. A query planner, an index walker, writing,
|
|
20
|
+
* journalling and transactions are most of SQLite and none of this, so none of
|
|
21
|
+
* them are here. Index b-tree pages are never visited: an index holds no data a
|
|
22
|
+
* table scan does not already produce.
|
|
23
|
+
*
|
|
24
|
+
* It exists because `runtime-resource` is browser-safe and ships no SQLite
|
|
25
|
+
* engine, so `.static` containers of the SQLite family previously needed a
|
|
26
|
+
* driver injected by the caller — which meant Node, and in practice the
|
|
27
|
+
* experimental `node:sqlite`. Nothing here touches a filesystem or a binding.
|
|
28
|
+
*
|
|
29
|
+
* Values map as the format defines them: NULL, integers, doubles, text decoded
|
|
30
|
+
* with the database's own encoding, and blobs as `Uint8Array`. An integer wider
|
|
31
|
+
* than `Number.MAX_SAFE_INTEGER` is returned as a `BigInt` rather than silently
|
|
32
|
+
* rounded.
|
|
33
|
+
*/
|
|
34
|
+
class CjsSqliteFormat {
|
|
35
|
+
/**
|
|
36
|
+
* Reports whether bytes are a SQLite container, by signature.
|
|
37
|
+
*
|
|
38
|
+
* @param {Uint8Array|ArrayBuffer} input Candidate bytes.
|
|
39
|
+
* @returns {boolean} True when the header signature matches.
|
|
40
|
+
*/
|
|
41
|
+
static is(input) {
|
|
42
|
+
try {
|
|
43
|
+
ReadHeader(Normalize(input));
|
|
44
|
+
return true;
|
|
45
|
+
} catch {
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Reports whether these bytes are a SQLite container, on the signature.
|
|
52
|
+
*
|
|
53
|
+
* The signature is the whole claim: the header either says
|
|
54
|
+
* `SQLite format 3` or it does not, so this is verified by content rather
|
|
55
|
+
* than inferred from a name — which matters because `.static` carries three
|
|
56
|
+
* unrelated families and the extension proves nothing.
|
|
57
|
+
*
|
|
58
|
+
* @param {ArrayBuffer|ArrayBufferView} input Container bytes.
|
|
59
|
+
* @param {object} [options] Probe options.
|
|
60
|
+
* @returns {object} Resource probe.
|
|
61
|
+
*/
|
|
62
|
+
static isSupported(input, options = null) {
|
|
63
|
+
return Probe(this.is(input), options);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// INTERIM, 2026-08-16. No `inspect` alias onto `isSupported`, while nineteen
|
|
67
|
+
// sibling formats have one. The capability-surface decision page names that
|
|
68
|
+
// alias as the confusion it wants removed, but its own "What is not proposed"
|
|
69
|
+
// says not to change things on its strength alone - so this is a divergence
|
|
70
|
+
// awaiting a decision, not a settled position.
|
|
71
|
+
//
|
|
72
|
+
// See maintenance/INTERIM-sqlite-and-static-2026-08-16.md before resolving it,
|
|
73
|
+
// and search for callers of `inspect` on any format first.
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Reads every table and row.
|
|
77
|
+
*
|
|
78
|
+
* @param {Uint8Array|ArrayBuffer} input Container bytes.
|
|
79
|
+
* @param {object} [options] Read options.
|
|
80
|
+
* @param {string} [options.output] `json`, `payload` or `raw`.
|
|
81
|
+
* @param {string[]} [options.tables] Read only these tables.
|
|
82
|
+
* @returns {object} Tables keyed by name, each an array of row objects.
|
|
83
|
+
*/
|
|
84
|
+
static read(input, options = {}) {
|
|
85
|
+
const output = options.output ?? OUTPUT_JSON;
|
|
86
|
+
const bytes = Normalize(input);
|
|
87
|
+
const header = ReadHeader(bytes);
|
|
88
|
+
const schema = ReadMaster(bytes, header);
|
|
89
|
+
if (output === OUTPUT_RAW) {
|
|
90
|
+
return {
|
|
91
|
+
header,
|
|
92
|
+
schema
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
const wanted = options.tables ? new Set(options.tables) : null;
|
|
96
|
+
const tables = {};
|
|
97
|
+
for (const entry of schema) {
|
|
98
|
+
if (entry.type !== "table" || wanted && !wanted.has(entry.name)) {
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// A table with no root page holds nothing: a view, or WITHOUT ROWID.
|
|
103
|
+
tables[entry.name] = entry.rootPage ? ReadTable(bytes, header, entry, output === OUTPUT_JSON) : [];
|
|
104
|
+
}
|
|
105
|
+
return tables;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Reads to plain JSON-compatible values, converting blobs to arrays. */
|
|
109
|
+
static readJSON(input, options = {}) {
|
|
110
|
+
return this.read(input, {
|
|
111
|
+
...options,
|
|
112
|
+
output: OUTPUT_JSON
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Reads with blobs left as `Uint8Array` and wide integers as `BigInt`. */
|
|
117
|
+
static readPayload(input, options = {}) {
|
|
118
|
+
return this.read(input, {
|
|
119
|
+
...options,
|
|
120
|
+
output: OUTPUT_PAYLOAD
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** Returns the parsed header and `sqlite_master` entries, decoding no rows. */
|
|
125
|
+
static readRaw(input, options = {}) {
|
|
126
|
+
return this.read(input, {
|
|
127
|
+
...options,
|
|
128
|
+
output: OUTPUT_RAW
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Lists the tables a container holds, without reading their rows.
|
|
134
|
+
*
|
|
135
|
+
* @param {Uint8Array|ArrayBuffer} input Container bytes.
|
|
136
|
+
* @returns {Array<object>} One entry per table.
|
|
137
|
+
*/
|
|
138
|
+
static tables(input) {
|
|
139
|
+
const bytes = Normalize(input);
|
|
140
|
+
return ReadMaster(bytes, ReadHeader(bytes)).filter(entry => entry.type === "table");
|
|
141
|
+
}
|
|
142
|
+
static Output = Object.freeze({
|
|
143
|
+
JSON: OUTPUT_JSON,
|
|
144
|
+
PAYLOAD: OUTPUT_PAYLOAD,
|
|
145
|
+
RAW: OUTPUT_RAW
|
|
146
|
+
});
|
|
147
|
+
static id = "sqlite";
|
|
148
|
+
static extensions = Object.freeze([".sqlite", ".db", ".sqlite3"]);
|
|
149
|
+
static type = Object.freeze(["data"]);
|
|
150
|
+
static mediaTypes = Object.freeze(["data"]);
|
|
151
|
+
static inputTypes = Object.freeze(["sqlite"]);
|
|
152
|
+
static outputTypes = Object.freeze([OUTPUT_JSON, OUTPUT_PAYLOAD]);
|
|
153
|
+
static debugOutputTypes = Object.freeze([OUTPUT_RAW]);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** Builds the probe shape the resource manager routes on. */
|
|
157
|
+
function Probe(recognized, options) {
|
|
158
|
+
return {
|
|
159
|
+
format: "sqlite",
|
|
160
|
+
source: "buffer",
|
|
161
|
+
supported: recognized ? "full" : "none",
|
|
162
|
+
confidence: 1,
|
|
163
|
+
preferred: "sqlite",
|
|
164
|
+
// The header signature is content, not a declaration, so a match is
|
|
165
|
+
// evidence rather than a claim to be checked later.
|
|
166
|
+
verified: true,
|
|
167
|
+
reason: recognized ? "Recognized a SQLite 3 container." : "The header signature is not SQLite 3.",
|
|
168
|
+
variants: [{
|
|
169
|
+
kind: "container",
|
|
170
|
+
codec: "sqlite",
|
|
171
|
+
supported: recognized
|
|
172
|
+
}],
|
|
173
|
+
metadata: {
|
|
174
|
+
family: "sqlite",
|
|
175
|
+
decodable: recognized
|
|
176
|
+
},
|
|
177
|
+
...(options || {})
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Reads `sqlite_master`, which is always the table b-tree rooted at page 1. */
|
|
182
|
+
function ReadMaster(bytes, header) {
|
|
183
|
+
const entries = [];
|
|
184
|
+
for (const row of TableRows(bytes, header, MASTER_ROOT_PAGE)) {
|
|
185
|
+
const [type, name, tableName, rootPage, sql] = ReadRecord(row.payload, header.encoding);
|
|
186
|
+
entries.push({
|
|
187
|
+
type: String(type ?? ""),
|
|
188
|
+
name: String(name ?? ""),
|
|
189
|
+
tableName: String(tableName ?? ""),
|
|
190
|
+
rootPage: Number(rootPage ?? 0),
|
|
191
|
+
sql: sql === null || sql === undefined ? "" : String(sql)
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
return entries;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Reads one table's rows as objects, named from its `CREATE TABLE` text. */
|
|
198
|
+
function ReadTable(bytes, header, entry, json) {
|
|
199
|
+
const {
|
|
200
|
+
names,
|
|
201
|
+
rowidAlias
|
|
202
|
+
} = ReadColumns(entry.sql);
|
|
203
|
+
const rows = [];
|
|
204
|
+
for (const row of TableRows(bytes, header, entry.rootPage)) {
|
|
205
|
+
const values = ReadRecord(row.payload, header.encoding);
|
|
206
|
+
const record = {};
|
|
207
|
+
for (let index = 0; index < values.length; index += 1) {
|
|
208
|
+
// Positional names keep a row readable when the statement could not be
|
|
209
|
+
// parsed, rather than dropping columns that certainly hold data.
|
|
210
|
+
const name = names[index] ?? `column${index}`;
|
|
211
|
+
const value = index === rowidAlias && values[index] === null ? row.rowid : values[index];
|
|
212
|
+
record[name] = json ? ToJSON(value) : value;
|
|
213
|
+
}
|
|
214
|
+
rows.push(record);
|
|
215
|
+
}
|
|
216
|
+
return rows;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** Converts a decoded value to something `JSON.stringify` keeps losslessly. */
|
|
220
|
+
function ToJSON(value) {
|
|
221
|
+
if (value instanceof Uint8Array) {
|
|
222
|
+
return Array.from(value);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// A BigInt has no JSON form; a decimal string keeps every digit.
|
|
226
|
+
return typeof value === "bigint" ? value.toString(10) : value;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** Accepts the byte forms every other format here accepts. */
|
|
230
|
+
function Normalize(input) {
|
|
231
|
+
if (input instanceof Uint8Array) {
|
|
232
|
+
return input;
|
|
233
|
+
}
|
|
234
|
+
if (input instanceof ArrayBuffer) {
|
|
235
|
+
return new Uint8Array(input);
|
|
236
|
+
}
|
|
237
|
+
if (ArrayBuffer.isView(input)) {
|
|
238
|
+
return new Uint8Array(input.buffer, input.byteOffset, input.byteLength);
|
|
239
|
+
}
|
|
240
|
+
throw SqliteError("A SQLite container must be supplied as bytes.");
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
export { CjsSqliteFormat };
|
|
244
|
+
//# sourceMappingURL=CjsSqliteFormat.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CjsSqliteFormat.js","sources":["../../../../src/formats/sqlite/CjsSqliteFormat.js"],"sourcesContent":["import { ReadColumns } from \"./core/sqliteSchema.js\";\nimport { ReadHeader, TableRows } from \"./core/sqlitePages.js\";\nimport { ReadRecord, SqliteError } from \"./core/sqliteRecords.js\";\n\nconst OUTPUT_JSON = \"json\";\nconst OUTPUT_PAYLOAD = \"payload\";\nconst OUTPUT_RAW = \"raw\";\nconst MASTER_ROOT_PAGE = 1;\n\n/**\n * Reads a SQLite 3 container as data.\n *\n * **This is a reader, not a database.** It accepts no SQL and runs no queries.\n * It returns the tables a container holds and every row of them, because that\n * is the whole of what this organization does with a SQLite file: the client's\n * `.static` containers and the prepared static data export are both opened,\n * scanned once, and closed.\n *\n * That scope is what makes it small. A query planner, an index walker, writing,\n * journalling and transactions are most of SQLite and none of this, so none of\n * them are here. Index b-tree pages are never visited: an index holds no data a\n * table scan does not already produce.\n *\n * It exists because `runtime-resource` is browser-safe and ships no SQLite\n * engine, so `.static` containers of the SQLite family previously needed a\n * driver injected by the caller — which meant Node, and in practice the\n * experimental `node:sqlite`. Nothing here touches a filesystem or a binding.\n *\n * Values map as the format defines them: NULL, integers, doubles, text decoded\n * with the database's own encoding, and blobs as `Uint8Array`. An integer wider\n * than `Number.MAX_SAFE_INTEGER` is returned as a `BigInt` rather than silently\n * rounded.\n */\nexport class CjsSqliteFormat {\n\n /**\n * Reports whether bytes are a SQLite container, by signature.\n *\n * @param {Uint8Array|ArrayBuffer} input Candidate bytes.\n * @returns {boolean} True when the header signature matches.\n */\n static is(input) {\n try {\n ReadHeader(Normalize(input));\n\n return true;\n } catch {\n return false;\n }\n }\n\n /**\n * Reports whether these bytes are a SQLite container, on the signature.\n *\n * The signature is the whole claim: the header either says\n * `SQLite format 3` or it does not, so this is verified by content rather\n * than inferred from a name — which matters because `.static` carries three\n * unrelated families and the extension proves nothing.\n *\n * @param {ArrayBuffer|ArrayBufferView} input Container bytes.\n * @param {object} [options] Probe options.\n * @returns {object} Resource probe.\n */\n static isSupported(input, options = null) {\n return Probe(this.is(input), options);\n }\n\n // INTERIM, 2026-08-16. No `inspect` alias onto `isSupported`, while nineteen\n // sibling formats have one. The capability-surface decision page names that\n // alias as the confusion it wants removed, but its own \"What is not proposed\"\n // says not to change things on its strength alone - so this is a divergence\n // awaiting a decision, not a settled position.\n //\n // See maintenance/INTERIM-sqlite-and-static-2026-08-16.md before resolving it,\n // and search for callers of `inspect` on any format first.\n\n /**\n * Reads every table and row.\n *\n * @param {Uint8Array|ArrayBuffer} input Container bytes.\n * @param {object} [options] Read options.\n * @param {string} [options.output] `json`, `payload` or `raw`.\n * @param {string[]} [options.tables] Read only these tables.\n * @returns {object} Tables keyed by name, each an array of row objects.\n */\n static read(input, options = {}) {\n const output = options.output ?? OUTPUT_JSON;\n const bytes = Normalize(input);\n const header = ReadHeader(bytes);\n const schema = ReadMaster(bytes, header);\n\n if (output === OUTPUT_RAW) {\n return { header, schema };\n }\n\n const wanted = options.tables ? new Set(options.tables) : null;\n const tables = {};\n\n for (const entry of schema) {\n if (entry.type !== \"table\" || (wanted && !wanted.has(entry.name))) {\n continue;\n }\n\n // A table with no root page holds nothing: a view, or WITHOUT ROWID.\n tables[entry.name] = entry.rootPage\n ? ReadTable(bytes, header, entry, output === OUTPUT_JSON)\n : [];\n }\n\n return tables;\n }\n\n /** Reads to plain JSON-compatible values, converting blobs to arrays. */\n static readJSON(input, options = {}) {\n return this.read(input, { ...options, output: OUTPUT_JSON });\n }\n\n /** Reads with blobs left as `Uint8Array` and wide integers as `BigInt`. */\n static readPayload(input, options = {}) {\n return this.read(input, { ...options, output: OUTPUT_PAYLOAD });\n }\n\n /** Returns the parsed header and `sqlite_master` entries, decoding no rows. */\n static readRaw(input, options = {}) {\n return this.read(input, { ...options, output: OUTPUT_RAW });\n }\n\n /**\n * Lists the tables a container holds, without reading their rows.\n *\n * @param {Uint8Array|ArrayBuffer} input Container bytes.\n * @returns {Array<object>} One entry per table.\n */\n static tables(input) {\n const bytes = Normalize(input);\n\n return ReadMaster(bytes, ReadHeader(bytes)).filter(entry => entry.type === \"table\");\n }\n\n static Output = Object.freeze({\n JSON: OUTPUT_JSON,\n PAYLOAD: OUTPUT_PAYLOAD,\n RAW: OUTPUT_RAW\n });\n\n static id = \"sqlite\";\n static extensions = Object.freeze([ \".sqlite\", \".db\", \".sqlite3\" ]);\n static type = Object.freeze([ \"data\" ]);\n static mediaTypes = Object.freeze([ \"data\" ]);\n static inputTypes = Object.freeze([ \"sqlite\" ]);\n static outputTypes = Object.freeze([ OUTPUT_JSON, OUTPUT_PAYLOAD ]);\n static debugOutputTypes = Object.freeze([ OUTPUT_RAW ]);\n}\n\n/** Builds the probe shape the resource manager routes on. */\nfunction Probe(recognized, options) {\n return {\n format: \"sqlite\",\n source: \"buffer\",\n supported: recognized ? \"full\" : \"none\",\n confidence: 1,\n preferred: \"sqlite\",\n // The header signature is content, not a declaration, so a match is\n // evidence rather than a claim to be checked later.\n verified: true,\n reason: recognized\n ? \"Recognized a SQLite 3 container.\"\n : \"The header signature is not SQLite 3.\",\n variants: [ { kind: \"container\", codec: \"sqlite\", supported: recognized } ],\n metadata: { family: \"sqlite\", decodable: recognized },\n ...(options || {})\n };\n}\n\n/** Reads `sqlite_master`, which is always the table b-tree rooted at page 1. */\nfunction ReadMaster(bytes, header) {\n const entries = [];\n\n for (const row of TableRows(bytes, header, MASTER_ROOT_PAGE)) {\n const [ type, name, tableName, rootPage, sql ] = ReadRecord(row.payload, header.encoding);\n\n entries.push({\n type: String(type ?? \"\"),\n name: String(name ?? \"\"),\n tableName: String(tableName ?? \"\"),\n rootPage: Number(rootPage ?? 0),\n sql: sql === null || sql === undefined ? \"\" : String(sql)\n });\n }\n\n return entries;\n}\n\n/** Reads one table's rows as objects, named from its `CREATE TABLE` text. */\nfunction ReadTable(bytes, header, entry, json) {\n const { names, rowidAlias } = ReadColumns(entry.sql);\n const rows = [];\n\n for (const row of TableRows(bytes, header, entry.rootPage)) {\n const values = ReadRecord(row.payload, header.encoding);\n const record = {};\n\n for (let index = 0; index < values.length; index += 1) {\n // Positional names keep a row readable when the statement could not be\n // parsed, rather than dropping columns that certainly hold data.\n const name = names[index] ?? `column${index}`;\n const value = index === rowidAlias && values[index] === null ? row.rowid : values[index];\n\n record[name] = json ? ToJSON(value) : value;\n }\n\n rows.push(record);\n }\n\n return rows;\n}\n\n/** Converts a decoded value to something `JSON.stringify` keeps losslessly. */\nfunction ToJSON(value) {\n if (value instanceof Uint8Array) {\n return Array.from(value);\n }\n\n // A BigInt has no JSON form; a decimal string keeps every digit.\n return typeof value === \"bigint\" ? value.toString(10) : value;\n}\n\n/** Accepts the byte forms every other format here accepts. */\nfunction Normalize(input) {\n if (input instanceof Uint8Array) {\n return input;\n }\n\n if (input instanceof ArrayBuffer) {\n return new Uint8Array(input);\n }\n\n if (ArrayBuffer.isView(input)) {\n return new Uint8Array(input.buffer, input.byteOffset, input.byteLength);\n }\n\n throw SqliteError(\"A SQLite container must be supplied as bytes.\");\n}\n\nexport default CjsSqliteFormat;\n"],"names":["OUTPUT_JSON","OUTPUT_PAYLOAD","OUTPUT_RAW","MASTER_ROOT_PAGE","CjsSqliteFormat","is","input","ReadHeader","Normalize","isSupported","options","Probe","read","output","bytes","header","schema","ReadMaster","wanted","tables","Set","entry","type","has","name","rootPage","ReadTable","readJSON","readPayload","readRaw","filter","Output","Object","freeze","JSON","PAYLOAD","RAW","id","extensions","mediaTypes","inputTypes","outputTypes","debugOutputTypes","recognized","format","source","supported","confidence","preferred","verified","reason","variants","kind","codec","metadata","family","decodable","entries","row","TableRows","tableName","sql","ReadRecord","payload","encoding","push","String","Number","undefined","json","names","rowidAlias","ReadColumns","rows","values","record","index","length","value","rowid","ToJSON","Uint8Array","Array","from","toString","ArrayBuffer","isView","buffer","byteOffset","byteLength","SqliteError"],"mappings":";;;;AAIA,MAAMA,WAAW,GAAG,MAAM;AAC1B,MAAMC,cAAc,GAAG,SAAS;AAChC,MAAMC,UAAU,GAAG,KAAK;AACxB,MAAMC,gBAAgB,GAAG,CAAC;;AAE1B;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,MAAMC,eAAe,CAAC;AAE3B;AACF;AACA;AACA;AACA;AACA;EACE,OAAOC,EAAEA,CAACC,KAAK,EAAE;IACf,IAAI;AACFC,MAAAA,UAAU,CAACC,SAAS,CAACF,KAAK,CAAC,CAAC;AAE5B,MAAA,OAAO,IAAI;AACb,IAAA,CAAC,CAAC,MAAM;AACN,MAAA,OAAO,KAAK;AACd,IAAA;AACF,EAAA;;AAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACE,EAAA,OAAOG,WAAWA,CAACH,KAAK,EAAEI,OAAO,GAAG,IAAI,EAAE;IACxC,OAAOC,KAAK,CAAC,IAAI,CAACN,EAAE,CAACC,KAAK,CAAC,EAAEI,OAAO,CAAC;AACvC,EAAA;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACE,OAAOE,IAAIA,CAACN,KAAK,EAAEI,OAAO,GAAG,EAAE,EAAE;AAC/B,IAAA,MAAMG,MAAM,GAAGH,OAAO,CAACG,MAAM,IAAIb,WAAW;AAC5C,IAAA,MAAMc,KAAK,GAAGN,SAAS,CAACF,KAAK,CAAC;AAC9B,IAAA,MAAMS,MAAM,GAAGR,UAAU,CAACO,KAAK,CAAC;AAChC,IAAA,MAAME,MAAM,GAAGC,UAAU,CAACH,KAAK,EAAEC,MAAM,CAAC;IAExC,IAAIF,MAAM,KAAKX,UAAU,EAAE;MACzB,OAAO;QAAEa,MAAM;AAAEC,QAAAA;OAAQ;AAC3B,IAAA;AAEA,IAAA,MAAME,MAAM,GAAGR,OAAO,CAACS,MAAM,GAAG,IAAIC,GAAG,CAACV,OAAO,CAACS,MAAM,CAAC,GAAG,IAAI;IAC9D,MAAMA,MAAM,GAAG,EAAE;AAEjB,IAAA,KAAK,MAAME,KAAK,IAAIL,MAAM,EAAE;AAC1B,MAAA,IAAIK,KAAK,CAACC,IAAI,KAAK,OAAO,IAAKJ,MAAM,IAAI,CAACA,MAAM,CAACK,GAAG,CAACF,KAAK,CAACG,IAAI,CAAE,EAAE;AACjE,QAAA;AACF,MAAA;;AAEA;MACAL,MAAM,CAACE,KAAK,CAACG,IAAI,CAAC,GAAGH,KAAK,CAACI,QAAQ,GAC/BC,SAAS,CAACZ,KAAK,EAAEC,MAAM,EAAEM,KAAK,EAAER,MAAM,KAAKb,WAAW,CAAC,GACvD,EAAE;AACR,IAAA;AAEA,IAAA,OAAOmB,MAAM;AACf,EAAA;;AAEA;EACA,OAAOQ,QAAQA,CAACrB,KAAK,EAAEI,OAAO,GAAG,EAAE,EAAE;AACnC,IAAA,OAAO,IAAI,CAACE,IAAI,CAACN,KAAK,EAAE;AAAE,MAAA,GAAGI,OAAO;AAAEG,MAAAA,MAAM,EAAEb;AAAY,KAAC,CAAC;AAC9D,EAAA;;AAEA;EACA,OAAO4B,WAAWA,CAACtB,KAAK,EAAEI,OAAO,GAAG,EAAE,EAAE;AACtC,IAAA,OAAO,IAAI,CAACE,IAAI,CAACN,KAAK,EAAE;AAAE,MAAA,GAAGI,OAAO;AAAEG,MAAAA,MAAM,EAAEZ;AAAe,KAAC,CAAC;AACjE,EAAA;;AAEA;EACA,OAAO4B,OAAOA,CAACvB,KAAK,EAAEI,OAAO,GAAG,EAAE,EAAE;AAClC,IAAA,OAAO,IAAI,CAACE,IAAI,CAACN,KAAK,EAAE;AAAE,MAAA,GAAGI,OAAO;AAAEG,MAAAA,MAAM,EAAEX;AAAW,KAAC,CAAC;AAC7D,EAAA;;AAEA;AACF;AACA;AACA;AACA;AACA;EACE,OAAOiB,MAAMA,CAACb,KAAK,EAAE;AACnB,IAAA,MAAMQ,KAAK,GAAGN,SAAS,CAACF,KAAK,CAAC;AAE9B,IAAA,OAAOW,UAAU,CAACH,KAAK,EAAEP,UAAU,CAACO,KAAK,CAAC,CAAC,CAACgB,MAAM,CAACT,KAAK,IAAIA,KAAK,CAACC,IAAI,KAAK,OAAO,CAAC;AACrF,EAAA;AAEA,EAAA,OAAOS,MAAM,GAAGC,MAAM,CAACC,MAAM,CAAC;AAC5BC,IAAAA,IAAI,EAAElC,WAAW;AACjBmC,IAAAA,OAAO,EAAElC,cAAc;AACvBmC,IAAAA,GAAG,EAAElC;AACP,GAAC,CAAC;EAEF,OAAOmC,EAAE,GAAG,QAAQ;AACpB,EAAA,OAAOC,UAAU,GAAGN,MAAM,CAACC,MAAM,CAAC,CAAE,SAAS,EAAE,KAAK,EAAE,UAAU,CAAE,CAAC;EACnE,OAAOX,IAAI,GAAGU,MAAM,CAACC,MAAM,CAAC,CAAE,MAAM,CAAE,CAAC;EACvC,OAAOM,UAAU,GAAGP,MAAM,CAACC,MAAM,CAAC,CAAE,MAAM,CAAE,CAAC;EAC7C,OAAOO,UAAU,GAAGR,MAAM,CAACC,MAAM,CAAC,CAAE,QAAQ,CAAE,CAAC;EAC/C,OAAOQ,WAAW,GAAGT,MAAM,CAACC,MAAM,CAAC,CAAEjC,WAAW,EAAEC,cAAc,CAAE,CAAC;EACnE,OAAOyC,gBAAgB,GAAGV,MAAM,CAACC,MAAM,CAAC,CAAE/B,UAAU,CAAE,CAAC;AACzD;;AAEA;AACA,SAASS,KAAKA,CAACgC,UAAU,EAAEjC,OAAO,EAAE;EAClC,OAAO;AACLkC,IAAAA,MAAM,EAAE,QAAQ;AAChBC,IAAAA,MAAM,EAAE,QAAQ;AAChBC,IAAAA,SAAS,EAAEH,UAAU,GAAG,MAAM,GAAG,MAAM;AACvCI,IAAAA,UAAU,EAAE,CAAC;AACbC,IAAAA,SAAS,EAAE,QAAQ;AACnB;AACA;AACAC,IAAAA,QAAQ,EAAE,IAAI;AACdC,IAAAA,MAAM,EAAEP,UAAU,GACd,kCAAkC,GAClC,uCAAuC;AAC3CQ,IAAAA,QAAQ,EAAE,CAAE;AAAEC,MAAAA,IAAI,EAAE,WAAW;AAAEC,MAAAA,KAAK,EAAE,QAAQ;AAAEP,MAAAA,SAAS,EAAEH;AAAW,KAAC,CAAE;AAC3EW,IAAAA,QAAQ,EAAE;AAAEC,MAAAA,MAAM,EAAE,QAAQ;AAAEC,MAAAA,SAAS,EAAEb;KAAY;IACrD,IAAIjC,OAAO,IAAI,EAAE;GAClB;AACH;;AAEA;AACA,SAASO,UAAUA,CAACH,KAAK,EAAEC,MAAM,EAAE;EACjC,MAAM0C,OAAO,GAAG,EAAE;EAElB,KAAK,MAAMC,GAAG,IAAIC,SAAS,CAAC7C,KAAK,EAAEC,MAAM,EAAEZ,gBAAgB,CAAC,EAAE;IAC5D,MAAM,CAAEmB,IAAI,EAAEE,IAAI,EAAEoC,SAAS,EAAEnC,QAAQ,EAAEoC,GAAG,CAAE,GAAGC,UAAU,CAACJ,GAAG,CAACK,OAAO,EAAEhD,MAAM,CAACiD,QAAQ,CAAC;IAEzFP,OAAO,CAACQ,IAAI,CAAC;AACX3C,MAAAA,IAAI,EAAE4C,MAAM,CAAC5C,IAAI,IAAI,EAAE,CAAC;AACxBE,MAAAA,IAAI,EAAE0C,MAAM,CAAC1C,IAAI,IAAI,EAAE,CAAC;AACxBoC,MAAAA,SAAS,EAAEM,MAAM,CAACN,SAAS,IAAI,EAAE,CAAC;AAClCnC,MAAAA,QAAQ,EAAE0C,MAAM,CAAC1C,QAAQ,IAAI,CAAC,CAAC;AAC/BoC,MAAAA,GAAG,EAAEA,GAAG,KAAK,IAAI,IAAIA,GAAG,KAAKO,SAAS,GAAG,EAAE,GAAGF,MAAM,CAACL,GAAG;AAC1D,KAAC,CAAC;AACJ,EAAA;AAEA,EAAA,OAAOJ,OAAO;AAChB;;AAEA;AACA,SAAS/B,SAASA,CAACZ,KAAK,EAAEC,MAAM,EAAEM,KAAK,EAAEgD,IAAI,EAAE;EAC7C,MAAM;IAAEC,KAAK;AAAEC,IAAAA;AAAW,GAAC,GAAGC,WAAW,CAACnD,KAAK,CAACwC,GAAG,CAAC;EACpD,MAAMY,IAAI,GAAG,EAAE;AAEf,EAAA,KAAK,MAAMf,GAAG,IAAIC,SAAS,CAAC7C,KAAK,EAAEC,MAAM,EAAEM,KAAK,CAACI,QAAQ,CAAC,EAAE;IAC1D,MAAMiD,MAAM,GAAGZ,UAAU,CAACJ,GAAG,CAACK,OAAO,EAAEhD,MAAM,CAACiD,QAAQ,CAAC;IACvD,MAAMW,MAAM,GAAG,EAAE;AAEjB,IAAA,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGF,MAAM,CAACG,MAAM,EAAED,KAAK,IAAI,CAAC,EAAE;AACrD;AACA;MACA,MAAMpD,IAAI,GAAG8C,KAAK,CAACM,KAAK,CAAC,IAAI,CAAA,MAAA,EAASA,KAAK,CAAA,CAAE;MAC7C,MAAME,KAAK,GAAGF,KAAK,KAAKL,UAAU,IAAIG,MAAM,CAACE,KAAK,CAAC,KAAK,IAAI,GAAGlB,GAAG,CAACqB,KAAK,GAAGL,MAAM,CAACE,KAAK,CAAC;MAExFD,MAAM,CAACnD,IAAI,CAAC,GAAG6C,IAAI,GAAGW,MAAM,CAACF,KAAK,CAAC,GAAGA,KAAK;AAC7C,IAAA;AAEAL,IAAAA,IAAI,CAACR,IAAI,CAACU,MAAM,CAAC;AACnB,EAAA;AAEA,EAAA,OAAOF,IAAI;AACb;;AAEA;AACA,SAASO,MAAMA,CAACF,KAAK,EAAE;EACrB,IAAIA,KAAK,YAAYG,UAAU,EAAE;AAC/B,IAAA,OAAOC,KAAK,CAACC,IAAI,CAACL,KAAK,CAAC;AAC1B,EAAA;;AAEA;AACA,EAAA,OAAO,OAAOA,KAAK,KAAK,QAAQ,GAAGA,KAAK,CAACM,QAAQ,CAAC,EAAE,CAAC,GAAGN,KAAK;AAC/D;;AAEA;AACA,SAAStE,SAASA,CAACF,KAAK,EAAE;EACxB,IAAIA,KAAK,YAAY2E,UAAU,EAAE;AAC/B,IAAA,OAAO3E,KAAK;AACd,EAAA;EAEA,IAAIA,KAAK,YAAY+E,WAAW,EAAE;AAChC,IAAA,OAAO,IAAIJ,UAAU,CAAC3E,KAAK,CAAC;AAC9B,EAAA;AAEA,EAAA,IAAI+E,WAAW,CAACC,MAAM,CAAChF,KAAK,CAAC,EAAE;AAC7B,IAAA,OAAO,IAAI2E,UAAU,CAAC3E,KAAK,CAACiF,MAAM,EAAEjF,KAAK,CAACkF,UAAU,EAAElF,KAAK,CAACmF,UAAU,CAAC;AACzE,EAAA;EAEA,MAAMC,WAAW,CAAC,+CAA+C,CAAC;AACpE;;;;"}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { SqliteError, ReadVarint } from './sqliteRecords.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* SQLite page and b-tree walking, for read-only full table scans.
|
|
5
|
+
*
|
|
6
|
+
* Only the table b-tree is walked. Index pages (`0x02` and `0x0a`) are never
|
|
7
|
+
* visited, because every caller of this format scans a table rather than
|
|
8
|
+
* looking a key up, and an index carries no data a table scan does not.
|
|
9
|
+
*/
|
|
10
|
+
const HEADER_SIZE = 100;
|
|
11
|
+
const SIGNATURE = "SQLite format 3\0";
|
|
12
|
+
const PAGE_INTERIOR_TABLE = 0x05;
|
|
13
|
+
const PAGE_LEAF_TABLE = 0x0d;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Reads and validates the hundred-byte database header.
|
|
17
|
+
*
|
|
18
|
+
* @param {Uint8Array} bytes Whole container.
|
|
19
|
+
* @returns {object} Header fields this reader needs.
|
|
20
|
+
*/
|
|
21
|
+
function ReadHeader(bytes) {
|
|
22
|
+
if (bytes.byteLength < HEADER_SIZE) {
|
|
23
|
+
throw SqliteError("A SQLite container is at least 100 bytes.");
|
|
24
|
+
}
|
|
25
|
+
const signature = new TextDecoder("latin1").decode(bytes.subarray(0, 16));
|
|
26
|
+
if (signature !== SIGNATURE) {
|
|
27
|
+
throw SqliteError("Not a SQLite container: the header signature does not match.");
|
|
28
|
+
}
|
|
29
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
30
|
+
const declared = view.getUint16(16, false);
|
|
31
|
+
|
|
32
|
+
// 1 is the documented spelling of 65536, which does not fit the field.
|
|
33
|
+
const pageSize = declared === 1 ? 65536 : declared;
|
|
34
|
+
if (pageSize < 512 || (pageSize & pageSize - 1) !== 0) {
|
|
35
|
+
throw SqliteError(`Invalid SQLite page size ${pageSize}.`);
|
|
36
|
+
}
|
|
37
|
+
const encoding = view.getUint32(56, false) || 1;
|
|
38
|
+
return {
|
|
39
|
+
pageSize,
|
|
40
|
+
reserved: view.getUint8(20),
|
|
41
|
+
encoding,
|
|
42
|
+
pageCount: view.getUint32(28, false),
|
|
43
|
+
usableSize: pageSize - view.getUint8(20)
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Returns the bytes of one page, numbered from 1 as the format numbers them.
|
|
49
|
+
*/
|
|
50
|
+
function Page(bytes, header, pageNumber) {
|
|
51
|
+
const start = (pageNumber - 1) * header.pageSize;
|
|
52
|
+
if (start < 0 || start + header.pageSize > bytes.byteLength) {
|
|
53
|
+
throw SqliteError(`Page ${pageNumber} is outside the container.`);
|
|
54
|
+
}
|
|
55
|
+
return bytes.subarray(start, start + header.pageSize);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Joins a cell payload with its overflow chain.
|
|
60
|
+
*
|
|
61
|
+
* The split point is the fiddly part of the format: a payload larger than the
|
|
62
|
+
* local maximum keeps `K` or `M` bytes on the page and puts the rest on a chain
|
|
63
|
+
* of overflow pages, each beginning with a four-byte next-page pointer. Getting
|
|
64
|
+
* this wrong truncates long values rather than failing, which is why the JSON
|
|
65
|
+
* documents in these containers are the case that exposes it.
|
|
66
|
+
*/
|
|
67
|
+
function ReadPayload(bytes, header, page, offset, size) {
|
|
68
|
+
const usable = header.usableSize;
|
|
69
|
+
const maxLocal = usable - 35;
|
|
70
|
+
let local = size;
|
|
71
|
+
if (size > maxLocal) {
|
|
72
|
+
const minLocal = Math.floor((usable - 12) * 32 / 255) - 23;
|
|
73
|
+
const K = minLocal + (size - minLocal) % (usable - 4);
|
|
74
|
+
local = K <= maxLocal ? K : minLocal;
|
|
75
|
+
}
|
|
76
|
+
const payload = new Uint8Array(size);
|
|
77
|
+
payload.set(page.subarray(offset, offset + local), 0);
|
|
78
|
+
if (local === size) {
|
|
79
|
+
// Wholly local, so there is no overflow pointer after it to read - and on
|
|
80
|
+
// the last cell of a page, reading one anyway runs off the end.
|
|
81
|
+
return payload;
|
|
82
|
+
}
|
|
83
|
+
let written = local;
|
|
84
|
+
let next = new DataView(page.buffer, page.byteOffset, page.byteLength).getUint32(offset + local, false);
|
|
85
|
+
while (written < size) {
|
|
86
|
+
if (!next) {
|
|
87
|
+
throw SqliteError("An overflow chain ended before the payload was complete.");
|
|
88
|
+
}
|
|
89
|
+
const overflow = Page(bytes, header, next);
|
|
90
|
+
const view = new DataView(overflow.buffer, overflow.byteOffset, overflow.byteLength);
|
|
91
|
+
const take = Math.min(usable - 4, size - written);
|
|
92
|
+
payload.set(overflow.subarray(4, 4 + take), written);
|
|
93
|
+
written += take;
|
|
94
|
+
next = view.getUint32(0, false);
|
|
95
|
+
}
|
|
96
|
+
return payload;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Walks one table b-tree and yields every leaf cell.
|
|
101
|
+
*
|
|
102
|
+
* @param {Uint8Array} bytes Whole container.
|
|
103
|
+
* @param {object} header Parsed database header.
|
|
104
|
+
* @param {number} rootPage Page number the table is rooted at.
|
|
105
|
+
* @yields {{ rowid: number, payload: Uint8Array }} Each row.
|
|
106
|
+
*/
|
|
107
|
+
function* TableRows(bytes, header, rootPage) {
|
|
108
|
+
const stack = [rootPage];
|
|
109
|
+
const seen = new Set();
|
|
110
|
+
while (stack.length) {
|
|
111
|
+
const pageNumber = stack.pop();
|
|
112
|
+
|
|
113
|
+
// A malformed or hostile container could point a page at itself.
|
|
114
|
+
if (seen.has(pageNumber)) {
|
|
115
|
+
throw SqliteError(`Page ${pageNumber} appears twice in one b-tree.`);
|
|
116
|
+
}
|
|
117
|
+
seen.add(pageNumber);
|
|
118
|
+
const page = Page(bytes, header, pageNumber);
|
|
119
|
+
// Page 1 carries the database header before its b-tree header.
|
|
120
|
+
const base = pageNumber === 1 ? HEADER_SIZE : 0;
|
|
121
|
+
const view = new DataView(page.buffer, page.byteOffset, page.byteLength);
|
|
122
|
+
const type = view.getUint8(base);
|
|
123
|
+
const cellCount = view.getUint16(base + 3, false);
|
|
124
|
+
if (type !== PAGE_LEAF_TABLE && type !== PAGE_INTERIOR_TABLE) {
|
|
125
|
+
throw SqliteError(`Page ${pageNumber} is not a table b-tree page (type ${type}).`);
|
|
126
|
+
}
|
|
127
|
+
const headerSize = type === PAGE_INTERIOR_TABLE ? 12 : 8;
|
|
128
|
+
const pointers = base + headerSize;
|
|
129
|
+
if (type === PAGE_INTERIOR_TABLE) {
|
|
130
|
+
// Children are in ascending key order, with the rightmost hanging off the
|
|
131
|
+
// page header rather than off a cell. They go on the stack backwards so
|
|
132
|
+
// that popping visits them in order: rows come out in rowid order, which
|
|
133
|
+
// is what every other SQLite reader produces and what a caller diffing
|
|
134
|
+
// two exports will assume.
|
|
135
|
+
const children = [];
|
|
136
|
+
for (let index = 0; index < cellCount; index += 1) {
|
|
137
|
+
children.push(view.getUint32(view.getUint16(pointers + index * 2, false), false));
|
|
138
|
+
}
|
|
139
|
+
children.push(view.getUint32(base + 8, false));
|
|
140
|
+
for (let index = children.length - 1; index >= 0; index -= 1) {
|
|
141
|
+
stack.push(children[index]);
|
|
142
|
+
}
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
for (let index = 0; index < cellCount; index += 1) {
|
|
146
|
+
const cell = view.getUint16(pointers + index * 2, false);
|
|
147
|
+
const size = ReadVarint(page, cell);
|
|
148
|
+
const rowid = ReadVarint(page, cell + size.size);
|
|
149
|
+
const start = cell + size.size + rowid.size;
|
|
150
|
+
yield {
|
|
151
|
+
rowid: rowid.value,
|
|
152
|
+
payload: ReadPayload(bytes, header, page, start, size.value)
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
export { HEADER_SIZE, ReadHeader, SIGNATURE, TableRows };
|
|
159
|
+
//# sourceMappingURL=sqlitePages.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sqlitePages.js","sources":["../../../../../src/formats/sqlite/core/sqlitePages.js"],"sourcesContent":["/**\n * SQLite page and b-tree walking, for read-only full table scans.\n *\n * Only the table b-tree is walked. Index pages (`0x02` and `0x0a`) are never\n * visited, because every caller of this format scans a table rather than\n * looking a key up, and an index carries no data a table scan does not.\n */\nimport { ReadVarint, SqliteError } from \"./sqliteRecords.js\";\n\nconst HEADER_SIZE = 100;\nconst SIGNATURE = \"SQLite format 3\\0\";\nconst PAGE_INTERIOR_TABLE = 0x05;\nconst PAGE_LEAF_TABLE = 0x0d;\n\n/**\n * Reads and validates the hundred-byte database header.\n *\n * @param {Uint8Array} bytes Whole container.\n * @returns {object} Header fields this reader needs.\n */\nexport function ReadHeader(bytes) {\n if (bytes.byteLength < HEADER_SIZE) {\n throw SqliteError(\"A SQLite container is at least 100 bytes.\");\n }\n\n const signature = new TextDecoder(\"latin1\").decode(bytes.subarray(0, 16));\n\n if (signature !== SIGNATURE) {\n throw SqliteError(\"Not a SQLite container: the header signature does not match.\");\n }\n\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n const declared = view.getUint16(16, false);\n\n // 1 is the documented spelling of 65536, which does not fit the field.\n const pageSize = declared === 1 ? 65536 : declared;\n\n if (pageSize < 512 || (pageSize & (pageSize - 1)) !== 0) {\n throw SqliteError(`Invalid SQLite page size ${pageSize}.`);\n }\n\n const encoding = view.getUint32(56, false) || 1;\n\n return {\n pageSize,\n reserved: view.getUint8(20),\n encoding,\n pageCount: view.getUint32(28, false),\n usableSize: pageSize - view.getUint8(20)\n };\n}\n\n/**\n * Returns the bytes of one page, numbered from 1 as the format numbers them.\n */\nfunction Page(bytes, header, pageNumber) {\n const start = (pageNumber - 1) * header.pageSize;\n\n if (start < 0 || start + header.pageSize > bytes.byteLength) {\n throw SqliteError(`Page ${pageNumber} is outside the container.`);\n }\n\n return bytes.subarray(start, start + header.pageSize);\n}\n\n/**\n * Joins a cell payload with its overflow chain.\n *\n * The split point is the fiddly part of the format: a payload larger than the\n * local maximum keeps `K` or `M` bytes on the page and puts the rest on a chain\n * of overflow pages, each beginning with a four-byte next-page pointer. Getting\n * this wrong truncates long values rather than failing, which is why the JSON\n * documents in these containers are the case that exposes it.\n */\nfunction ReadPayload(bytes, header, page, offset, size) {\n const usable = header.usableSize;\n const maxLocal = usable - 35;\n let local = size;\n\n if (size > maxLocal) {\n const minLocal = Math.floor(((usable - 12) * 32) / 255) - 23;\n const K = minLocal + ((size - minLocal) % (usable - 4));\n\n local = K <= maxLocal ? K : minLocal;\n }\n\n const payload = new Uint8Array(size);\n\n payload.set(page.subarray(offset, offset + local), 0);\n\n if (local === size) {\n // Wholly local, so there is no overflow pointer after it to read - and on\n // the last cell of a page, reading one anyway runs off the end.\n return payload;\n }\n\n let written = local;\n let next = new DataView(page.buffer, page.byteOffset, page.byteLength)\n .getUint32(offset + local, false);\n\n while (written < size) {\n if (!next) {\n throw SqliteError(\"An overflow chain ended before the payload was complete.\");\n }\n\n const overflow = Page(bytes, header, next);\n const view = new DataView(overflow.buffer, overflow.byteOffset, overflow.byteLength);\n const take = Math.min(usable - 4, size - written);\n\n payload.set(overflow.subarray(4, 4 + take), written);\n written += take;\n next = view.getUint32(0, false);\n }\n\n return payload;\n}\n\n/**\n * Walks one table b-tree and yields every leaf cell.\n *\n * @param {Uint8Array} bytes Whole container.\n * @param {object} header Parsed database header.\n * @param {number} rootPage Page number the table is rooted at.\n * @yields {{ rowid: number, payload: Uint8Array }} Each row.\n */\nexport function* TableRows(bytes, header, rootPage) {\n const stack = [ rootPage ];\n const seen = new Set();\n\n while (stack.length) {\n const pageNumber = stack.pop();\n\n // A malformed or hostile container could point a page at itself.\n if (seen.has(pageNumber)) {\n throw SqliteError(`Page ${pageNumber} appears twice in one b-tree.`);\n }\n\n seen.add(pageNumber);\n\n const page = Page(bytes, header, pageNumber);\n // Page 1 carries the database header before its b-tree header.\n const base = pageNumber === 1 ? HEADER_SIZE : 0;\n const view = new DataView(page.buffer, page.byteOffset, page.byteLength);\n const type = view.getUint8(base);\n const cellCount = view.getUint16(base + 3, false);\n\n if (type !== PAGE_LEAF_TABLE && type !== PAGE_INTERIOR_TABLE) {\n throw SqliteError(`Page ${pageNumber} is not a table b-tree page (type ${type}).`);\n }\n\n const headerSize = type === PAGE_INTERIOR_TABLE ? 12 : 8;\n const pointers = base + headerSize;\n\n if (type === PAGE_INTERIOR_TABLE) {\n // Children are in ascending key order, with the rightmost hanging off the\n // page header rather than off a cell. They go on the stack backwards so\n // that popping visits them in order: rows come out in rowid order, which\n // is what every other SQLite reader produces and what a caller diffing\n // two exports will assume.\n const children = [];\n\n for (let index = 0; index < cellCount; index += 1) {\n children.push(view.getUint32(view.getUint16(pointers + index * 2, false), false));\n }\n\n children.push(view.getUint32(base + 8, false));\n\n for (let index = children.length - 1; index >= 0; index -= 1) {\n stack.push(children[index]);\n }\n\n continue;\n }\n\n for (let index = 0; index < cellCount; index += 1) {\n const cell = view.getUint16(pointers + index * 2, false);\n const size = ReadVarint(page, cell);\n const rowid = ReadVarint(page, cell + size.size);\n const start = cell + size.size + rowid.size;\n\n yield {\n rowid: rowid.value,\n payload: ReadPayload(bytes, header, page, start, size.value)\n };\n }\n }\n}\n\nexport { HEADER_SIZE, SIGNATURE };\n"],"names":["HEADER_SIZE","SIGNATURE","PAGE_INTERIOR_TABLE","PAGE_LEAF_TABLE","ReadHeader","bytes","byteLength","SqliteError","signature","TextDecoder","decode","subarray","view","DataView","buffer","byteOffset","declared","getUint16","pageSize","encoding","getUint32","reserved","getUint8","pageCount","usableSize","Page","header","pageNumber","start","ReadPayload","page","offset","size","usable","maxLocal","local","minLocal","Math","floor","K","payload","Uint8Array","set","written","next","overflow","take","min","TableRows","rootPage","stack","seen","Set","length","pop","has","add","base","type","cellCount","headerSize","pointers","children","index","push","cell","ReadVarint","rowid","value"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AAGA,MAAMA,WAAW,GAAG;AACpB,MAAMC,SAAS,GAAG;AAClB,MAAMC,mBAAmB,GAAG,IAAI;AAChC,MAAMC,eAAe,GAAG,IAAI;;AAE5B;AACA;AACA;AACA;AACA;AACA;AACO,SAASC,UAAUA,CAACC,KAAK,EAAE;AAChC,EAAA,IAAIA,KAAK,CAACC,UAAU,GAAGN,WAAW,EAAE;IAClC,MAAMO,WAAW,CAAC,2CAA2C,CAAC;AAChE,EAAA;AAEA,EAAA,MAAMC,SAAS,GAAG,IAAIC,WAAW,CAAC,QAAQ,CAAC,CAACC,MAAM,CAACL,KAAK,CAACM,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;EAEzE,IAAIH,SAAS,KAAKP,SAAS,EAAE;IAC3B,MAAMM,WAAW,CAAC,8DAA8D,CAAC;AACnF,EAAA;AAEA,EAAA,MAAMK,IAAI,GAAG,IAAIC,QAAQ,CAACR,KAAK,CAACS,MAAM,EAAET,KAAK,CAACU,UAAU,EAAEV,KAAK,CAACC,UAAU,CAAC;EAC3E,MAAMU,QAAQ,GAAGJ,IAAI,CAACK,SAAS,CAAC,EAAE,EAAE,KAAK,CAAC;;AAE1C;EACA,MAAMC,QAAQ,GAAGF,QAAQ,KAAK,CAAC,GAAG,KAAK,GAAGA,QAAQ;AAElD,EAAA,IAAIE,QAAQ,GAAG,GAAG,IAAI,CAACA,QAAQ,GAAIA,QAAQ,GAAG,CAAE,MAAM,CAAC,EAAE;AACvD,IAAA,MAAMX,WAAW,CAAC,CAAA,yBAAA,EAA4BW,QAAQ,GAAG,CAAC;AAC5D,EAAA;EAEA,MAAMC,QAAQ,GAAGP,IAAI,CAACQ,SAAS,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,CAAC;EAE/C,OAAO;IACLF,QAAQ;AACRG,IAAAA,QAAQ,EAAET,IAAI,CAACU,QAAQ,CAAC,EAAE,CAAC;IAC3BH,QAAQ;IACRI,SAAS,EAAEX,IAAI,CAACQ,SAAS,CAAC,EAAE,EAAE,KAAK,CAAC;AACpCI,IAAAA,UAAU,EAAEN,QAAQ,GAAGN,IAAI,CAACU,QAAQ,CAAC,EAAE;GACxC;AACH;;AAEA;AACA;AACA;AACA,SAASG,IAAIA,CAACpB,KAAK,EAAEqB,MAAM,EAAEC,UAAU,EAAE;EACvC,MAAMC,KAAK,GAAG,CAACD,UAAU,GAAG,CAAC,IAAID,MAAM,CAACR,QAAQ;AAEhD,EAAA,IAAIU,KAAK,GAAG,CAAC,IAAIA,KAAK,GAAGF,MAAM,CAACR,QAAQ,GAAGb,KAAK,CAACC,UAAU,EAAE;AAC3D,IAAA,MAAMC,WAAW,CAAC,CAAA,KAAA,EAAQoB,UAAU,4BAA4B,CAAC;AACnE,EAAA;EAEA,OAAOtB,KAAK,CAACM,QAAQ,CAACiB,KAAK,EAAEA,KAAK,GAAGF,MAAM,CAACR,QAAQ,CAAC;AACvD;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASW,WAAWA,CAACxB,KAAK,EAAEqB,MAAM,EAAEI,IAAI,EAAEC,MAAM,EAAEC,IAAI,EAAE;AACtD,EAAA,MAAMC,MAAM,GAAGP,MAAM,CAACF,UAAU;AAChC,EAAA,MAAMU,QAAQ,GAAGD,MAAM,GAAG,EAAE;EAC5B,IAAIE,KAAK,GAAGH,IAAI;EAEhB,IAAIA,IAAI,GAAGE,QAAQ,EAAE;AACnB,IAAA,MAAME,QAAQ,GAAGC,IAAI,CAACC,KAAK,CAAE,CAACL,MAAM,GAAG,EAAE,IAAI,EAAE,GAAI,GAAG,CAAC,GAAG,EAAE;AAC5D,IAAA,MAAMM,CAAC,GAAGH,QAAQ,GAAI,CAACJ,IAAI,GAAGI,QAAQ,KAAKH,MAAM,GAAG,CAAC,CAAE;AAEvDE,IAAAA,KAAK,GAAGI,CAAC,IAAIL,QAAQ,GAAGK,CAAC,GAAGH,QAAQ;AACtC,EAAA;AAEA,EAAA,MAAMI,OAAO,GAAG,IAAIC,UAAU,CAACT,IAAI,CAAC;AAEpCQ,EAAAA,OAAO,CAACE,GAAG,CAACZ,IAAI,CAACnB,QAAQ,CAACoB,MAAM,EAAEA,MAAM,GAAGI,KAAK,CAAC,EAAE,CAAC,CAAC;EAErD,IAAIA,KAAK,KAAKH,IAAI,EAAE;AAClB;AACA;AACA,IAAA,OAAOQ,OAAO;AAChB,EAAA;EAEA,IAAIG,OAAO,GAAGR,KAAK;EACnB,IAAIS,IAAI,GAAG,IAAI/B,QAAQ,CAACiB,IAAI,CAAChB,MAAM,EAAEgB,IAAI,CAACf,UAAU,EAAEe,IAAI,CAACxB,UAAU,CAAC,CACnEc,SAAS,CAACW,MAAM,GAAGI,KAAK,EAAE,KAAK,CAAC;EAEnC,OAAOQ,OAAO,GAAGX,IAAI,EAAE;IACrB,IAAI,CAACY,IAAI,EAAE;MACT,MAAMrC,WAAW,CAAC,0DAA0D,CAAC;AAC/E,IAAA;IAEA,MAAMsC,QAAQ,GAAGpB,IAAI,CAACpB,KAAK,EAAEqB,MAAM,EAAEkB,IAAI,CAAC;AAC1C,IAAA,MAAMhC,IAAI,GAAG,IAAIC,QAAQ,CAACgC,QAAQ,CAAC/B,MAAM,EAAE+B,QAAQ,CAAC9B,UAAU,EAAE8B,QAAQ,CAACvC,UAAU,CAAC;AACpF,IAAA,MAAMwC,IAAI,GAAGT,IAAI,CAACU,GAAG,CAACd,MAAM,GAAG,CAAC,EAAED,IAAI,GAAGW,OAAO,CAAC;AAEjDH,IAAAA,OAAO,CAACE,GAAG,CAACG,QAAQ,CAAClC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAGmC,IAAI,CAAC,EAAEH,OAAO,CAAC;AACpDA,IAAAA,OAAO,IAAIG,IAAI;IACfF,IAAI,GAAGhC,IAAI,CAACQ,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC;AACjC,EAAA;AAEA,EAAA,OAAOoB,OAAO;AAChB;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,UAAUQ,SAASA,CAAC3C,KAAK,EAAEqB,MAAM,EAAEuB,QAAQ,EAAE;AAClD,EAAA,MAAMC,KAAK,GAAG,CAAED,QAAQ,CAAE;AAC1B,EAAA,MAAME,IAAI,GAAG,IAAIC,GAAG,EAAE;EAEtB,OAAOF,KAAK,CAACG,MAAM,EAAE;AACnB,IAAA,MAAM1B,UAAU,GAAGuB,KAAK,CAACI,GAAG,EAAE;;AAE9B;AACA,IAAA,IAAIH,IAAI,CAACI,GAAG,CAAC5B,UAAU,CAAC,EAAE;AACxB,MAAA,MAAMpB,WAAW,CAAC,CAAA,KAAA,EAAQoB,UAAU,+BAA+B,CAAC;AACtE,IAAA;AAEAwB,IAAAA,IAAI,CAACK,GAAG,CAAC7B,UAAU,CAAC;IAEpB,MAAMG,IAAI,GAAGL,IAAI,CAACpB,KAAK,EAAEqB,MAAM,EAAEC,UAAU,CAAC;AAC5C;IACA,MAAM8B,IAAI,GAAG9B,UAAU,KAAK,CAAC,GAAG3B,WAAW,GAAG,CAAC;AAC/C,IAAA,MAAMY,IAAI,GAAG,IAAIC,QAAQ,CAACiB,IAAI,CAAChB,MAAM,EAAEgB,IAAI,CAACf,UAAU,EAAEe,IAAI,CAACxB,UAAU,CAAC;AACxE,IAAA,MAAMoD,IAAI,GAAG9C,IAAI,CAACU,QAAQ,CAACmC,IAAI,CAAC;IAChC,MAAME,SAAS,GAAG/C,IAAI,CAACK,SAAS,CAACwC,IAAI,GAAG,CAAC,EAAE,KAAK,CAAC;AAEjD,IAAA,IAAIC,IAAI,KAAKvD,eAAe,IAAIuD,IAAI,KAAKxD,mBAAmB,EAAE;AAC5D,MAAA,MAAMK,WAAW,CAAC,CAAA,KAAA,EAAQoB,UAAU,CAAA,kCAAA,EAAqC+B,IAAI,IAAI,CAAC;AACpF,IAAA;IAEA,MAAME,UAAU,GAAGF,IAAI,KAAKxD,mBAAmB,GAAG,EAAE,GAAG,CAAC;AACxD,IAAA,MAAM2D,QAAQ,GAAGJ,IAAI,GAAGG,UAAU;IAElC,IAAIF,IAAI,KAAKxD,mBAAmB,EAAE;AAChC;AACA;AACA;AACA;AACA;MACA,MAAM4D,QAAQ,GAAG,EAAE;AAEnB,MAAA,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGJ,SAAS,EAAEI,KAAK,IAAI,CAAC,EAAE;QACjDD,QAAQ,CAACE,IAAI,CAACpD,IAAI,CAACQ,SAAS,CAACR,IAAI,CAACK,SAAS,CAAC4C,QAAQ,GAAGE,KAAK,GAAG,CAAC,EAAE,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;AACnF,MAAA;AAEAD,MAAAA,QAAQ,CAACE,IAAI,CAACpD,IAAI,CAACQ,SAAS,CAACqC,IAAI,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;AAE9C,MAAA,KAAK,IAAIM,KAAK,GAAGD,QAAQ,CAACT,MAAM,GAAG,CAAC,EAAEU,KAAK,IAAI,CAAC,EAAEA,KAAK,IAAI,CAAC,EAAE;AAC5Db,QAAAA,KAAK,CAACc,IAAI,CAACF,QAAQ,CAACC,KAAK,CAAC,CAAC;AAC7B,MAAA;AAEA,MAAA;AACF,IAAA;AAEA,IAAA,KAAK,IAAIA,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGJ,SAAS,EAAEI,KAAK,IAAI,CAAC,EAAE;AACjD,MAAA,MAAME,IAAI,GAAGrD,IAAI,CAACK,SAAS,CAAC4C,QAAQ,GAAGE,KAAK,GAAG,CAAC,EAAE,KAAK,CAAC;AACxD,MAAA,MAAM/B,IAAI,GAAGkC,UAAU,CAACpC,IAAI,EAAEmC,IAAI,CAAC;MACnC,MAAME,KAAK,GAAGD,UAAU,CAACpC,IAAI,EAAEmC,IAAI,GAAGjC,IAAI,CAACA,IAAI,CAAC;MAChD,MAAMJ,KAAK,GAAGqC,IAAI,GAAGjC,IAAI,CAACA,IAAI,GAAGmC,KAAK,CAACnC,IAAI;MAE3C,MAAM;QACJmC,KAAK,EAAEA,KAAK,CAACC,KAAK;AAClB5B,QAAAA,OAAO,EAAEX,WAAW,CAACxB,KAAK,EAAEqB,MAAM,EAAEI,IAAI,EAAEF,KAAK,EAAEI,IAAI,CAACoC,KAAK;OAC5D;AACH,IAAA;AACF,EAAA;AACF;;;;"}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQLite record and varint decoding.
|
|
3
|
+
*
|
|
4
|
+
* The file format is documented by SQLite itself; this decodes the read-only
|
|
5
|
+
* subset a full table scan needs and nothing else.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
const TEXT_DECODERS = Object.freeze({
|
|
9
|
+
1: "utf-8",
|
|
10
|
+
2: "utf-16le",
|
|
11
|
+
3: "utf-16be"
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Reads a big-endian base-128 varint, of one to nine bytes.
|
|
16
|
+
*
|
|
17
|
+
* A ninth byte exists only when all eight before it set their continuation bit,
|
|
18
|
+
* and it contributes all eight of its own bits rather than seven. Treating the
|
|
19
|
+
* eighth byte as that terminator instead - which this did until it was
|
|
20
|
+
* checked - returns size 9 for an eight-byte varint, so the caller skips a byte
|
|
21
|
+
* it never read. No container in the corpus holds a varint long enough to
|
|
22
|
+
* expose it, which is why decoding 80,765 rows correctly proved nothing here.
|
|
23
|
+
*
|
|
24
|
+
* @param {Uint8Array} bytes Buffer to read from.
|
|
25
|
+
* @param {number} offset Byte offset.
|
|
26
|
+
* @returns {{ value: number, size: number }} Value and bytes consumed.
|
|
27
|
+
*/
|
|
28
|
+
function ReadVarint(bytes, offset) {
|
|
29
|
+
let value = 0n;
|
|
30
|
+
|
|
31
|
+
// The first eight bytes carry seven bits each, high bit continuing.
|
|
32
|
+
for (let index = 0; index < 8; index += 1) {
|
|
33
|
+
const byte = bytes[offset + index];
|
|
34
|
+
if (byte === undefined) {
|
|
35
|
+
throw SqliteError("A varint runs past the end of the container.");
|
|
36
|
+
}
|
|
37
|
+
value = value << 7n | BigInt(byte & 0x7f);
|
|
38
|
+
if ((byte & 0x80) === 0) {
|
|
39
|
+
return {
|
|
40
|
+
value: Narrow(value),
|
|
41
|
+
size: index + 1
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Only when all eight continued does a ninth byte exist, and it contributes
|
|
47
|
+
// all eight of its bits rather than seven - the one irregularity in the
|
|
48
|
+
// encoding, and 8x7 + 8 is where the full 64 bits come from.
|
|
49
|
+
const ninth = bytes[offset + 8];
|
|
50
|
+
if (ninth === undefined) {
|
|
51
|
+
throw SqliteError("A nine-byte varint runs past the end of the container.");
|
|
52
|
+
}
|
|
53
|
+
value = value << 8n | BigInt(ninth);
|
|
54
|
+
return {
|
|
55
|
+
value: Narrow(BigInt.asIntN(64, value)),
|
|
56
|
+
size: 9
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Returns a Number when one holds the value exactly, and the BigInt otherwise.
|
|
62
|
+
*
|
|
63
|
+
* A varint reaches 64 bits, so it can carry a rowid past 2^53 that a Number
|
|
64
|
+
* would silently round. Rounding a key is worse than handing back a type the
|
|
65
|
+
* caller has to notice, and it matches how a record's own wide integers are
|
|
66
|
+
* already returned.
|
|
67
|
+
*
|
|
68
|
+
* Sizes and header lengths never come close, so callers doing arithmetic on
|
|
69
|
+
* those keep seeing Numbers.
|
|
70
|
+
*
|
|
71
|
+
* @param {bigint} value Decoded value.
|
|
72
|
+
* @returns {number|bigint} Narrowed value.
|
|
73
|
+
*/
|
|
74
|
+
function Narrow(value) {
|
|
75
|
+
const narrowed = Number(value);
|
|
76
|
+
return Number.isSafeInteger(narrowed) ? narrowed : value;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Returns the byte length a serial type occupies in a record body.
|
|
81
|
+
*
|
|
82
|
+
* @param {number} serial Serial type code.
|
|
83
|
+
* @returns {number} Byte length.
|
|
84
|
+
*/
|
|
85
|
+
function SerialSize(serial) {
|
|
86
|
+
if (serial >= 12) {
|
|
87
|
+
return Math.floor((serial - (serial % 2 === 0 ? 12 : 13)) / 2);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// 0 and 8/9 are constants stored entirely in the type; 7 is a double.
|
|
91
|
+
return [0, 1, 2, 3, 4, 6, 8, 8, 0, 0][serial] ?? 0;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Decodes one serial-typed value from a record body.
|
|
96
|
+
*
|
|
97
|
+
* @param {Uint8Array} body Record body bytes.
|
|
98
|
+
* @param {number} offset Offset within the body.
|
|
99
|
+
* @param {number} serial Serial type code.
|
|
100
|
+
* @param {number} encoding Database text encoding (1, 2 or 3).
|
|
101
|
+
* @returns {*} Decoded value.
|
|
102
|
+
*/
|
|
103
|
+
function ReadSerial(body, offset, serial, encoding) {
|
|
104
|
+
if (serial === 0) {
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
if (serial === 8) {
|
|
108
|
+
return 0;
|
|
109
|
+
}
|
|
110
|
+
if (serial === 9) {
|
|
111
|
+
return 1;
|
|
112
|
+
}
|
|
113
|
+
if (serial >= 1 && serial <= 6) {
|
|
114
|
+
const size = SerialSize(serial);
|
|
115
|
+
let value = 0n;
|
|
116
|
+
for (let index = 0; index < size; index += 1) {
|
|
117
|
+
value = value << 8n | BigInt(body[offset + index]);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// These are all signed, so a leading high bit means a negative number.
|
|
121
|
+
const signed = BigInt.asIntN(size * 8, value);
|
|
122
|
+
return Number.isSafeInteger(Number(signed)) ? Number(signed) : signed;
|
|
123
|
+
}
|
|
124
|
+
if (serial === 7) {
|
|
125
|
+
return new DataView(body.buffer, body.byteOffset + offset, 8).getFloat64(0, false);
|
|
126
|
+
}
|
|
127
|
+
const size = SerialSize(serial);
|
|
128
|
+
const slice = body.subarray(offset, offset + size);
|
|
129
|
+
|
|
130
|
+
// Even codes from 12 are blobs; odd codes from 13 are text.
|
|
131
|
+
if (serial % 2 === 0) {
|
|
132
|
+
return slice.slice();
|
|
133
|
+
}
|
|
134
|
+
const label = TEXT_DECODERS[encoding];
|
|
135
|
+
if (!label) {
|
|
136
|
+
throw SqliteError(`Unsupported text encoding ${encoding}.`);
|
|
137
|
+
}
|
|
138
|
+
return new TextDecoder(label).decode(slice);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Decodes a complete record into its column values.
|
|
143
|
+
*
|
|
144
|
+
* @param {Uint8Array} payload Record bytes, overflow already joined.
|
|
145
|
+
* @param {number} encoding Database text encoding.
|
|
146
|
+
* @returns {Array<*>} Column values in order.
|
|
147
|
+
*/
|
|
148
|
+
function ReadRecord(payload, encoding) {
|
|
149
|
+
const header = ReadVarint(payload, 0);
|
|
150
|
+
const serials = [];
|
|
151
|
+
let cursor = header.size;
|
|
152
|
+
while (cursor < header.value) {
|
|
153
|
+
const serial = ReadVarint(payload, cursor);
|
|
154
|
+
serials.push(serial.value);
|
|
155
|
+
cursor += serial.size;
|
|
156
|
+
}
|
|
157
|
+
const values = [];
|
|
158
|
+
let body = header.value;
|
|
159
|
+
for (const serial of serials) {
|
|
160
|
+
values.push(ReadSerial(payload, body, serial, encoding));
|
|
161
|
+
body += SerialSize(serial);
|
|
162
|
+
}
|
|
163
|
+
return values;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** Creates a tagged error, so a caller can tell a malformed container apart. */
|
|
167
|
+
function SqliteError(message) {
|
|
168
|
+
const error = new Error(message);
|
|
169
|
+
error.code = "CJS_SQLITE_INVALID";
|
|
170
|
+
return error;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export { ReadRecord, ReadSerial, ReadVarint, SerialSize, SqliteError };
|
|
174
|
+
//# sourceMappingURL=sqliteRecords.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sqliteRecords.js","sources":["../../../../../src/formats/sqlite/core/sqliteRecords.js"],"sourcesContent":["/**\n * SQLite record and varint decoding.\n *\n * The file format is documented by SQLite itself; this decodes the read-only\n * subset a full table scan needs and nothing else.\n */\n\nconst TEXT_DECODERS = Object.freeze({\n 1: \"utf-8\",\n 2: \"utf-16le\",\n 3: \"utf-16be\"\n});\n\n/**\n * Reads a big-endian base-128 varint, of one to nine bytes.\n *\n * A ninth byte exists only when all eight before it set their continuation bit,\n * and it contributes all eight of its own bits rather than seven. Treating the\n * eighth byte as that terminator instead - which this did until it was\n * checked - returns size 9 for an eight-byte varint, so the caller skips a byte\n * it never read. No container in the corpus holds a varint long enough to\n * expose it, which is why decoding 80,765 rows correctly proved nothing here.\n *\n * @param {Uint8Array} bytes Buffer to read from.\n * @param {number} offset Byte offset.\n * @returns {{ value: number, size: number }} Value and bytes consumed.\n */\nexport function ReadVarint(bytes, offset) {\n let value = 0n;\n\n // The first eight bytes carry seven bits each, high bit continuing.\n for (let index = 0; index < 8; index += 1) {\n const byte = bytes[offset + index];\n\n if (byte === undefined) {\n throw SqliteError(\"A varint runs past the end of the container.\");\n }\n\n value = (value << 7n) | BigInt(byte & 0x7f);\n\n if ((byte & 0x80) === 0) {\n return { value: Narrow(value), size: index + 1 };\n }\n }\n\n // Only when all eight continued does a ninth byte exist, and it contributes\n // all eight of its bits rather than seven - the one irregularity in the\n // encoding, and 8x7 + 8 is where the full 64 bits come from.\n const ninth = bytes[offset + 8];\n\n if (ninth === undefined) {\n throw SqliteError(\"A nine-byte varint runs past the end of the container.\");\n }\n\n value = (value << 8n) | BigInt(ninth);\n\n return { value: Narrow(BigInt.asIntN(64, value)), size: 9 };\n}\n\n/**\n * Returns a Number when one holds the value exactly, and the BigInt otherwise.\n *\n * A varint reaches 64 bits, so it can carry a rowid past 2^53 that a Number\n * would silently round. Rounding a key is worse than handing back a type the\n * caller has to notice, and it matches how a record's own wide integers are\n * already returned.\n *\n * Sizes and header lengths never come close, so callers doing arithmetic on\n * those keep seeing Numbers.\n *\n * @param {bigint} value Decoded value.\n * @returns {number|bigint} Narrowed value.\n */\nfunction Narrow(value) {\n const narrowed = Number(value);\n\n return Number.isSafeInteger(narrowed) ? narrowed : value;\n}\n\n/**\n * Returns the byte length a serial type occupies in a record body.\n *\n * @param {number} serial Serial type code.\n * @returns {number} Byte length.\n */\nexport function SerialSize(serial) {\n if (serial >= 12) {\n return Math.floor((serial - (serial % 2 === 0 ? 12 : 13)) / 2);\n }\n\n // 0 and 8/9 are constants stored entirely in the type; 7 is a double.\n return [ 0, 1, 2, 3, 4, 6, 8, 8, 0, 0 ][serial] ?? 0;\n}\n\n/**\n * Decodes one serial-typed value from a record body.\n *\n * @param {Uint8Array} body Record body bytes.\n * @param {number} offset Offset within the body.\n * @param {number} serial Serial type code.\n * @param {number} encoding Database text encoding (1, 2 or 3).\n * @returns {*} Decoded value.\n */\nexport function ReadSerial(body, offset, serial, encoding) {\n if (serial === 0) {\n return null;\n }\n\n if (serial === 8) {\n return 0;\n }\n\n if (serial === 9) {\n return 1;\n }\n\n if (serial >= 1 && serial <= 6) {\n const size = SerialSize(serial);\n let value = 0n;\n\n for (let index = 0; index < size; index += 1) {\n value = (value << 8n) | BigInt(body[offset + index]);\n }\n\n // These are all signed, so a leading high bit means a negative number.\n const signed = BigInt.asIntN(size * 8, value);\n\n return Number.isSafeInteger(Number(signed)) ? Number(signed) : signed;\n }\n\n if (serial === 7) {\n return new DataView(body.buffer, body.byteOffset + offset, 8).getFloat64(0, false);\n }\n\n const size = SerialSize(serial);\n const slice = body.subarray(offset, offset + size);\n\n // Even codes from 12 are blobs; odd codes from 13 are text.\n if (serial % 2 === 0) {\n return slice.slice();\n }\n\n const label = TEXT_DECODERS[encoding];\n\n if (!label) {\n throw SqliteError(`Unsupported text encoding ${encoding}.`);\n }\n\n return new TextDecoder(label).decode(slice);\n}\n\n/**\n * Decodes a complete record into its column values.\n *\n * @param {Uint8Array} payload Record bytes, overflow already joined.\n * @param {number} encoding Database text encoding.\n * @returns {Array<*>} Column values in order.\n */\nexport function ReadRecord(payload, encoding) {\n const header = ReadVarint(payload, 0);\n const serials = [];\n let cursor = header.size;\n\n while (cursor < header.value) {\n const serial = ReadVarint(payload, cursor);\n\n serials.push(serial.value);\n cursor += serial.size;\n }\n\n const values = [];\n let body = header.value;\n\n for (const serial of serials) {\n values.push(ReadSerial(payload, body, serial, encoding));\n body += SerialSize(serial);\n }\n\n return values;\n}\n\n/** Creates a tagged error, so a caller can tell a malformed container apart. */\nexport function SqliteError(message) {\n const error = new Error(message);\n\n error.code = \"CJS_SQLITE_INVALID\";\n\n return error;\n}\n"],"names":["TEXT_DECODERS","Object","freeze","ReadVarint","bytes","offset","value","index","byte","undefined","SqliteError","BigInt","Narrow","size","ninth","asIntN","narrowed","Number","isSafeInteger","SerialSize","serial","Math","floor","ReadSerial","body","encoding","signed","DataView","buffer","byteOffset","getFloat64","slice","subarray","label","TextDecoder","decode","ReadRecord","payload","header","serials","cursor","push","values","message","error","Error","code"],"mappings":"AAAA;AACA;AACA;AACA;AACA;AACA;;AAEA,MAAMA,aAAa,GAAGC,MAAM,CAACC,MAAM,CAAC;AAClC,EAAA,CAAC,EAAE,OAAO;AACV,EAAA,CAAC,EAAE,UAAU;AACb,EAAA,CAAC,EAAE;AACL,CAAC,CAAC;;AAEF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,SAASC,UAAUA,CAACC,KAAK,EAAEC,MAAM,EAAE;EACxC,IAAIC,KAAK,GAAG,EAAE;;AAEd;AACA,EAAA,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAG,CAAC,EAAEA,KAAK,IAAI,CAAC,EAAE;AACzC,IAAA,MAAMC,IAAI,GAAGJ,KAAK,CAACC,MAAM,GAAGE,KAAK,CAAC;IAElC,IAAIC,IAAI,KAAKC,SAAS,EAAE;MACtB,MAAMC,WAAW,CAAC,8CAA8C,CAAC;AACnE,IAAA;IAEAJ,KAAK,GAAIA,KAAK,IAAI,EAAE,GAAIK,MAAM,CAACH,IAAI,GAAG,IAAI,CAAC;AAE3C,IAAA,IAAI,CAACA,IAAI,GAAG,IAAI,MAAM,CAAC,EAAE;MACvB,OAAO;AAAEF,QAAAA,KAAK,EAAEM,MAAM,CAACN,KAAK,CAAC;QAAEO,IAAI,EAAEN,KAAK,GAAG;OAAG;AAClD,IAAA;AACF,EAAA;;AAEA;AACA;AACA;AACA,EAAA,MAAMO,KAAK,GAAGV,KAAK,CAACC,MAAM,GAAG,CAAC,CAAC;EAE/B,IAAIS,KAAK,KAAKL,SAAS,EAAE;IACvB,MAAMC,WAAW,CAAC,wDAAwD,CAAC;AAC7E,EAAA;EAEAJ,KAAK,GAAIA,KAAK,IAAI,EAAE,GAAIK,MAAM,CAACG,KAAK,CAAC;EAErC,OAAO;IAAER,KAAK,EAAEM,MAAM,CAACD,MAAM,CAACI,MAAM,CAAC,EAAE,EAAET,KAAK,CAAC,CAAC;AAAEO,IAAAA,IAAI,EAAE;GAAG;AAC7D;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASD,MAAMA,CAACN,KAAK,EAAE;AACrB,EAAA,MAAMU,QAAQ,GAAGC,MAAM,CAACX,KAAK,CAAC;EAE9B,OAAOW,MAAM,CAACC,aAAa,CAACF,QAAQ,CAAC,GAAGA,QAAQ,GAAGV,KAAK;AAC1D;;AAEA;AACA;AACA;AACA;AACA;AACA;AACO,SAASa,UAAUA,CAACC,MAAM,EAAE;EACjC,IAAIA,MAAM,IAAI,EAAE,EAAE;AAChB,IAAA,OAAOC,IAAI,CAACC,KAAK,CAAC,CAACF,MAAM,IAAIA,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;AAChE,EAAA;;AAEA;EACA,OAAO,CAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAE,CAACA,MAAM,CAAC,IAAI,CAAC;AACtD;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,SAASG,UAAUA,CAACC,IAAI,EAAEnB,MAAM,EAAEe,MAAM,EAAEK,QAAQ,EAAE;EACzD,IAAIL,MAAM,KAAK,CAAC,EAAE;AAChB,IAAA,OAAO,IAAI;AACb,EAAA;EAEA,IAAIA,MAAM,KAAK,CAAC,EAAE;AAChB,IAAA,OAAO,CAAC;AACV,EAAA;EAEA,IAAIA,MAAM,KAAK,CAAC,EAAE;AAChB,IAAA,OAAO,CAAC;AACV,EAAA;AAEA,EAAA,IAAIA,MAAM,IAAI,CAAC,IAAIA,MAAM,IAAI,CAAC,EAAE;AAC9B,IAAA,MAAMP,IAAI,GAAGM,UAAU,CAACC,MAAM,CAAC;IAC/B,IAAId,KAAK,GAAG,EAAE;AAEd,IAAA,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGM,IAAI,EAAEN,KAAK,IAAI,CAAC,EAAE;AAC5CD,MAAAA,KAAK,GAAIA,KAAK,IAAI,EAAE,GAAIK,MAAM,CAACa,IAAI,CAACnB,MAAM,GAAGE,KAAK,CAAC,CAAC;AACtD,IAAA;;AAEA;IACA,MAAMmB,MAAM,GAAGf,MAAM,CAACI,MAAM,CAACF,IAAI,GAAG,CAAC,EAAEP,KAAK,CAAC;AAE7C,IAAA,OAAOW,MAAM,CAACC,aAAa,CAACD,MAAM,CAACS,MAAM,CAAC,CAAC,GAAGT,MAAM,CAACS,MAAM,CAAC,GAAGA,MAAM;AACvE,EAAA;EAEA,IAAIN,MAAM,KAAK,CAAC,EAAE;IAChB,OAAO,IAAIO,QAAQ,CAACH,IAAI,CAACI,MAAM,EAAEJ,IAAI,CAACK,UAAU,GAAGxB,MAAM,EAAE,CAAC,CAAC,CAACyB,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC;AACpF,EAAA;AAEA,EAAA,MAAMjB,IAAI,GAAGM,UAAU,CAACC,MAAM,CAAC;EAC/B,MAAMW,KAAK,GAAGP,IAAI,CAACQ,QAAQ,CAAC3B,MAAM,EAAEA,MAAM,GAAGQ,IAAI,CAAC;;AAElD;AACA,EAAA,IAAIO,MAAM,GAAG,CAAC,KAAK,CAAC,EAAE;AACpB,IAAA,OAAOW,KAAK,CAACA,KAAK,EAAE;AACtB,EAAA;AAEA,EAAA,MAAME,KAAK,GAAGjC,aAAa,CAACyB,QAAQ,CAAC;EAErC,IAAI,CAACQ,KAAK,EAAE;AACV,IAAA,MAAMvB,WAAW,CAAC,CAAA,0BAAA,EAA6Be,QAAQ,GAAG,CAAC;AAC7D,EAAA;EAEA,OAAO,IAAIS,WAAW,CAACD,KAAK,CAAC,CAACE,MAAM,CAACJ,KAAK,CAAC;AAC7C;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,SAASK,UAAUA,CAACC,OAAO,EAAEZ,QAAQ,EAAE;AAC5C,EAAA,MAAMa,MAAM,GAAGnC,UAAU,CAACkC,OAAO,EAAE,CAAC,CAAC;EACrC,MAAME,OAAO,GAAG,EAAE;AAClB,EAAA,IAAIC,MAAM,GAAGF,MAAM,CAACzB,IAAI;AAExB,EAAA,OAAO2B,MAAM,GAAGF,MAAM,CAAChC,KAAK,EAAE;AAC5B,IAAA,MAAMc,MAAM,GAAGjB,UAAU,CAACkC,OAAO,EAAEG,MAAM,CAAC;AAE1CD,IAAAA,OAAO,CAACE,IAAI,CAACrB,MAAM,CAACd,KAAK,CAAC;IAC1BkC,MAAM,IAAIpB,MAAM,CAACP,IAAI;AACvB,EAAA;EAEA,MAAM6B,MAAM,GAAG,EAAE;AACjB,EAAA,IAAIlB,IAAI,GAAGc,MAAM,CAAChC,KAAK;AAEvB,EAAA,KAAK,MAAMc,MAAM,IAAImB,OAAO,EAAE;AAC5BG,IAAAA,MAAM,CAACD,IAAI,CAAClB,UAAU,CAACc,OAAO,EAAEb,IAAI,EAAEJ,MAAM,EAAEK,QAAQ,CAAC,CAAC;AACxDD,IAAAA,IAAI,IAAIL,UAAU,CAACC,MAAM,CAAC;AAC5B,EAAA;AAEA,EAAA,OAAOsB,MAAM;AACf;;AAEA;AACO,SAAShC,WAAWA,CAACiC,OAAO,EAAE;AACnC,EAAA,MAAMC,KAAK,GAAG,IAAIC,KAAK,CAACF,OAAO,CAAC;EAEhCC,KAAK,CAACE,IAAI,GAAG,oBAAoB;AAEjC,EAAA,OAAOF,KAAK;AACd;;;;"}
|