@zaisma/game-server 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bench/glass.js +85 -8
- package/bin/node-parts.js +2 -1
- package/package.json +1 -1
- package/src/database.d.ts +21 -8
- package/src/database.js +57 -15
- package/src/dump.d.ts +4 -1
- package/src/dump.js +166 -18
package/bench/glass.js
CHANGED
|
@@ -6255,6 +6255,7 @@ var ERROR_CODES = {
|
|
|
6255
6255
|
bad_values: { httpStatus: 400, meaning: "values isn't a list of text, numbers, null and { blob_base64 } for the SQL's ?s." },
|
|
6256
6256
|
bad_player_id: { httpStatus: 400, meaning: "player_id isn't a player ID (player_\u2026)." },
|
|
6257
6257
|
too_many_rows: { httpStatus: 400, meaning: "A statement answered more than 10,000 rows." },
|
|
6258
|
+
integer_too_large: { httpStatus: 400, meaning: "A statement answered a whole number past 2^53, which JavaScript can't hold exactly." },
|
|
6258
6259
|
answer_too_big: { httpStatus: 400, meaning: "The rows answered come to more than 8 MB." },
|
|
6259
6260
|
database_full: { httpStatus: 403, meaning: "The statement would grow the game's storage (its database and kept rooms) past the 10 GB a game can keep." },
|
|
6260
6261
|
database_unavailable: { httpStatus: 503, meaning: "The game's database didn't answer." },
|
|
@@ -7634,6 +7635,30 @@ var MAXIMUM_DATABASE_SIZE_BYTES = 10 * 1024 ** 3;
|
|
|
7634
7635
|
|
|
7635
7636
|
// ../core/src/hosting.ts
|
|
7636
7637
|
var DAY_MS = 24 * 36e5;
|
|
7638
|
+
var USAGE_FIELDS = {
|
|
7639
|
+
room_play_duration_sec: "roomPlayDurationSec",
|
|
7640
|
+
room_awake_duration_sec: "roomAwakeDurationSec",
|
|
7641
|
+
message_count: "inboundMessageCount",
|
|
7642
|
+
room_connection_count: "roomConnectionCount",
|
|
7643
|
+
object_request_count: "objectRequestCount",
|
|
7644
|
+
bookkeeping_written_row_count: "bookkeepingWrittenRowCount",
|
|
7645
|
+
relayed_size_bytes: "relayedSizeBytes",
|
|
7646
|
+
game_server_cpu_duration_ms: "roomCpuDurationMs",
|
|
7647
|
+
outside_room_cpu_duration_ms: "outsideRoomCpuDurationMs",
|
|
7648
|
+
game_server_startup_cpu_duration_ms: "startupCpuDurationMs",
|
|
7649
|
+
outside_call_count: "outsideCallCount",
|
|
7650
|
+
game_server_session_count: "gameServerSessionCount",
|
|
7651
|
+
log_file_count: "logFileCount",
|
|
7652
|
+
game_server_request_count: "requestCount",
|
|
7653
|
+
game_server_scheduled_run_count: "scheduledRunCount",
|
|
7654
|
+
object_awake_duration_sec: "objectAwakeDurationSec",
|
|
7655
|
+
database_written_row_count: "writtenRowCount",
|
|
7656
|
+
database_read_row_count: "readRowCount",
|
|
7657
|
+
database_call_count: "databaseCallCount",
|
|
7658
|
+
database_awake_duration_sec: "databaseAwakeDurationSec",
|
|
7659
|
+
adult_chat_check_count: "adultChatCheckCount"
|
|
7660
|
+
};
|
|
7661
|
+
var USAGE_COLUMNS = Object.keys(USAGE_FIELDS);
|
|
7637
7662
|
|
|
7638
7663
|
// ../core/src/forum.ts
|
|
7639
7664
|
var FORUM_SECTIONS = [
|
|
@@ -7714,12 +7739,16 @@ var OUTSIDE_CALL_LIMITS = {
|
|
|
7714
7739
|
|
|
7715
7740
|
// src/database.ts
|
|
7716
7741
|
var DATABASE_LIMITS = {
|
|
7717
|
-
/** One statement's SQL. */
|
|
7742
|
+
/** One statement's SQL, in characters. */
|
|
7718
7743
|
maximumSqlCharacterCount: 1e5,
|
|
7744
|
+
/** One statement's SQL, in UTF-8 bytes: Cloudflare's SQLite takes 100 KB, so text in other scripts is held to it too. */
|
|
7745
|
+
maximumSqlSizeBytes: 1e5,
|
|
7719
7746
|
/** Values for one statement's ?s. */
|
|
7720
7747
|
maximumValueCount: 100,
|
|
7721
|
-
/** One text or blob value
|
|
7748
|
+
/** One text or blob value a game server or query binds. */
|
|
7722
7749
|
maximumValueSizeBytes: 1024 * 1024,
|
|
7750
|
+
/** One text or blob value in an import, which brings back every row an export wrote: what the database holds in a value. */
|
|
7751
|
+
maximumImportValueSizeBytes: 2 * 1024 * 1024,
|
|
7723
7752
|
/** Statements in one batch. */
|
|
7724
7753
|
maximumStatementCount: 100,
|
|
7725
7754
|
/** Rows one statement may answer; past it, page with LIMIT and OFFSET or a WHERE. */
|
|
@@ -10499,7 +10528,7 @@ var LIGHT_GLASS = `color-scheme: light;
|
|
|
10499
10528
|
--divider: 1px solid var(--line);
|
|
10500
10529
|
--toggle-off: rgba(60, 72, 94, 0.2); --check-press: rgba(60, 72, 94, 0.16); --row-hover: rgba(255, 255, 255, 0.24); --row-press: rgba(20, 23, 28, 0.05); --tag-new: rgba(34, 160, 90, 0.26); --tag-new-ink: #0A5A2C; --tag-warning: rgba(217, 165, 28, 0.3); --tag-warning-ink: var(--warning); --tag-plain: rgba(60, 72, 94, 0.1); --tag-carve: inset 0 1px 2px rgba(20, 30, 50, 0.22), inset 0 -1px 0 rgba(255, 255, 255, 0.75); --face-ring: rgba(255, 255, 255, 0.85); --presence-ring: #fff; --online: #3BB273; --picture-shadow: rgba(20, 23, 28, 0.3);
|
|
10501
10530
|
/* Data: a table's header row over the rows scrolling under it, a chart's fill under its line, a meter's fill and its amber past 80%. */
|
|
10502
|
-
--table-head: rgba(250, 252, 255, 0.9); --table-row-head: rgba(244, 246, 250, 0.97); --chart-fill: rgba(20, 23, 28, 0.07); --chart-bar: rgba(60, 72, 94, 0.26); --chart-ring: rgba(250, 252, 255, 0.9); --chart-tip: rgba(250, 252, 255, 0.94); --meter-fill: rgba(60, 72, 94, 0.42); --meter-near: #D9A51C; --meter-full: #A87C0E; --meter-full-ink: var(--warning);
|
|
10531
|
+
--table-head: rgba(250, 252, 255, 0.9); --table-row-head: rgba(244, 246, 250, 0.97); --chart-fill: rgba(20, 23, 28, 0.07); --chart-bar: rgba(60, 72, 94, 0.26); --chart-ring: rgba(250, 252, 255, 0.9); --chart-tip: rgba(250, 252, 255, 0.94); --meter-fill: rgba(60, 72, 94, 0.42); --meter-near: #D9A51C; --meter-full: #A87C0E; --meter-full-ink: var(--warning); --part-0: #3E6FD8; --part-1: #7B5BD6; --part-2: #1E9C87; --part-3: #5E8C2E; --part-4: #D2577F; --part-5: #2A8FB8;
|
|
10503
10532
|
/* A file drop's dashed outline, waiting for a file. */
|
|
10504
10533
|
--drop-dash: rgba(20, 23, 28, 0.3);
|
|
10505
10534
|
--rail-line: rgba(20, 23, 28, 0.2); --rail-thumb: rgba(24, 28, 36, 0.55); --rail-edge: rgba(255, 255, 255, 0.55); --rail-mark: rgba(20, 23, 28, 0.45);
|
|
@@ -10535,7 +10564,7 @@ var SMOKED_GLASS = `color-scheme: dark;
|
|
|
10535
10564
|
--glass-btn-lift: inset 0 -12px 18px -12px rgba(0, 0, 0, 0.5), inset 0 1px 0 rgba(255, 255, 255, 0.22); --glass-btn-hair: rgba(26, 44, 80, 0.16); --glass-btn-hair-rest: rgba(26, 44, 80, 0.28);
|
|
10536
10565
|
--clear-hover: rgba(255, 255, 255, 0.08); --clear-press: rgba(255, 255, 255, 0.04); --clear-glass-mark: #fff; --on-light: none; --on-smoked: inline; --divider: 1px solid var(--line);
|
|
10537
10566
|
--toggle-off: rgba(0, 0, 0, 0.34); --check-press: rgba(0, 0, 0, 0.3); --row-hover: rgba(255, 255, 255, 0.1); --row-press: rgba(255, 255, 255, 0.22); --tag-new: rgba(30, 150, 80, 0.32); --tag-new-ink: #A6F0C6; --tag-warning: rgba(230, 181, 58, 0.24); --tag-warning-ink: var(--warning); --tag-plain: rgba(0, 0, 0, 0.34); --tag-carve: inset 0 1.5px 3px rgba(0, 0, 0, 0.8), inset 0 -1px 0 rgba(255, 255, 255, 0.24); --face-ring: rgba(6, 8, 12, 0.55); --presence-ring: #1A1D24; --online: #3BB273; --picture-shadow: rgba(0, 0, 0, 0.55);
|
|
10538
|
-
--table-head: rgba(14, 17, 24, 0.9); --table-row-head: rgba(24, 28, 36, 0.97); --chart-fill: rgba(255, 255, 255, 0.08); --chart-bar: rgba(255, 255, 255, 0.3); --chart-ring: rgba(20, 23, 30, 0.9); --chart-tip: rgba(28, 32, 40, 0.94); --meter-fill: rgba(255, 255, 255, 0.6); --meter-near: #E6B53A; --meter-full: #A9811C; --meter-full-ink: var(--warning); --drop-dash: rgba(255, 255, 255, 0.34);
|
|
10567
|
+
--table-head: rgba(14, 17, 24, 0.9); --table-row-head: rgba(24, 28, 36, 0.97); --chart-fill: rgba(255, 255, 255, 0.08); --chart-bar: rgba(255, 255, 255, 0.3); --chart-ring: rgba(20, 23, 30, 0.9); --chart-tip: rgba(28, 32, 40, 0.94); --meter-fill: rgba(255, 255, 255, 0.6); --meter-near: #E6B53A; --meter-full: #A9811C; --meter-full-ink: var(--warning); --part-0: #6E97F0; --part-1: #A08AF0; --part-2: #3CC4AC; --part-3: #8DBA55; --part-4: #EC7FA3; --part-5: #5BC0E3; --drop-dash: rgba(255, 255, 255, 0.34);
|
|
10539
10568
|
--rail-line: rgba(255, 255, 255, 0.2); --rail-thumb: rgba(255, 255, 255, 0.55); --rail-edge: rgba(255, 255, 255, 0.35); --rail-mark: rgba(255, 255, 255, 0.5);
|
|
10540
10569
|
/* A light rim on dark glass stands out far more than on light glass, so smoked glass draws it fainter than light glass's. It has no caustics:
|
|
10541
10570
|
on light glass they're light on light and only round the edge off, but on dark glass they read as a glow fanning in from the rim, which
|
|
@@ -15947,17 +15976,24 @@ function meterState(options) {
|
|
|
15947
15976
|
}
|
|
15948
15977
|
function usageTrack(options) {
|
|
15949
15978
|
const { sharePct, tone } = meterState(options);
|
|
15950
|
-
return html`<span class="usage${tone}">${meterTrack(options, sharePct)}</span>`;
|
|
15979
|
+
return html`<span class="usage${tone}">${meterTrack(options, sharePct)}${usageKey(options)}</span>`;
|
|
15951
15980
|
}
|
|
15952
15981
|
function meterTrack(options, sharePct) {
|
|
15953
|
-
|
|
15982
|
+
const parts2 = options.parts?.filter((part) => part.sharePct > 0) ?? [];
|
|
15983
|
+
const fill = parts2.length ? html`<span class="usage-fill has-parts" data-cut-line="4" style="width:${sharePct.toFixed(2)}%">${parts2.map((part) => html`<span class="usage-part" data-part="${String(part.partColorIndex)}" style="flex-grow:${String(part.sharePct)}"></span>`)}</span>` : html`<span class="usage-fill" data-cut-line="4" style="width:${sharePct.toFixed(2)}%"></span>`;
|
|
15984
|
+
return html`<div class="usage-track cut-carved" data-cut-line="6" role="meter" aria-label="${options.label}" aria-valuemin="0" aria-valuemax="${options.limitNumber}" aria-valuenow="${Math.min(options.usedNumber, options.limitNumber)}">${CUT_LINE}${fill}</div>`;
|
|
15985
|
+
}
|
|
15986
|
+
function usageKey(options) {
|
|
15987
|
+
const parts2 = options.parts?.filter((part) => part.sharePct > 0) ?? [];
|
|
15988
|
+
if (!parts2.length) return "";
|
|
15989
|
+
return html`<ul class="usage-key" aria-label="What used ${options.label}">${parts2.map((part) => html`<li data-part="${String(part.partColorIndex)}"><span class="usage-swatch" aria-hidden="true"></span>${part.label}<b>${String(part.sharePct)}%</b></li>`)}</ul>`;
|
|
15954
15990
|
}
|
|
15955
15991
|
function usageMeter(options) {
|
|
15956
15992
|
const { sharePct, isAllUsed, tone } = meterState(options);
|
|
15957
15993
|
const overNumber = options.usedNumber - options.limitNumber;
|
|
15958
15994
|
const allUsedWords = overNumber > 0 ? `, ${chartWords(overNumber)} ${options.unit} over` : ", all used";
|
|
15959
15995
|
const resetWords = options.resetDate ? `Resets ${(/* @__PURE__ */ new Date(`${options.resetDate}T00:00:00Z`)).toLocaleDateString("en-US", { month: "short", day: "numeric", timeZone: "UTC" })}.` : "";
|
|
15960
|
-
return html`<div class="usage${tone}"><span class="usage-label">${isAllUsed ? icon("warning") : ""}${options.label}</span><p class="usage-numbers"><span class="usage-fact">${options.shownWords ?? html`${chartWords(options.usedNumber)} of ${chartWords(options.limitNumber)}\u00a0${options.unit}`}${isAllUsed ? html`<b>${allUsedWords}</b>` : ""}${resetWords ? "\xA0\xB7" : ""}</span>${resetWords ? html` <span class="usage-fact">${resetWords}</span>` : ""}</p>${meterTrack(options, sharePct)}</div>`;
|
|
15996
|
+
return html`<div class="usage${tone}"><span class="usage-label">${isAllUsed ? icon("warning") : ""}${options.label}</span><p class="usage-numbers"><span class="usage-fact">${options.shownWords ?? html`${chartWords(options.usedNumber)} of ${chartWords(options.limitNumber)}\u00a0${options.unit}`}${isAllUsed ? html`<b>${allUsedWords}</b>` : ""}${resetWords ? "\xA0\xB7" : ""}</span>${resetWords ? html` <span class="usage-fact">${resetWords}</span>` : ""}</p>${usageKey(options)}${meterTrack(options, sharePct)}</div>`;
|
|
15961
15997
|
}
|
|
15962
15998
|
var meterPane = (options) => glassPane({ paneWidth: "fit", body: usageMeter(options) });
|
|
15963
15999
|
component({
|
|
@@ -15981,6 +16017,24 @@ component({
|
|
|
15981
16017
|
.usage:is(.is-used, .is-near) .usage-fill { min-width: 6px; }
|
|
15982
16018
|
.usage.is-near .usage-fill { background: var(--meter-near); }
|
|
15983
16019
|
.usage.is-full .usage-fill { background: repeating-linear-gradient(-45deg, var(--meter-near) 0 6px, var(--meter-full) 6px 12px); }
|
|
16020
|
+
/* With parts, the fill is what used it, side by side in each part's color, a hairline of the track between them; past 80% the colors
|
|
16021
|
+
stay and the numbers say so, and all used, amber stripes run across them. The key names each part with its share: over the track on a
|
|
16022
|
+
meter of its own, so the track stays its foot, and under it in a list row or another plate, where the row's lines are over it. Each
|
|
16023
|
+
swatch hangs before its words, so the words keep the margin. */
|
|
16024
|
+
.usage-fill.has-parts, .usage:is(.is-near, .is-full) .usage-fill.has-parts { display: flex; gap: var(--gap-control-row); background: none; }
|
|
16025
|
+
.usage-part { display: block; min-width: 3px; background: var(--part-color); }
|
|
16026
|
+
.usage.is-full .usage-part { background: repeating-linear-gradient(-45deg, var(--part-color) 0 6px, var(--meter-full) 6px 12px); }
|
|
16027
|
+
.usage [data-part="0"] { --part-color: var(--part-0); }
|
|
16028
|
+
.usage [data-part="1"] { --part-color: var(--part-1); }
|
|
16029
|
+
.usage [data-part="2"] { --part-color: var(--part-2); }
|
|
16030
|
+
.usage [data-part="3"] { --part-color: var(--part-3); }
|
|
16031
|
+
.usage [data-part="4"] { --part-color: var(--part-4); }
|
|
16032
|
+
.usage [data-part="5"] { --part-color: var(--part-5); }
|
|
16033
|
+
.usage-key { display: flex; flex-wrap: wrap; gap: var(--gap-words-words) var(--gap-block-block); list-style: none; margin: 0; padding: 0 var(--pad-words); }
|
|
16034
|
+
.usage-key li { position: relative; display: inline-flex; align-items: center; gap: var(--gap-words-words); font: var(--type-facts); color: var(--ink-2); }
|
|
16035
|
+
.usage-key .usage-swatch { position: absolute; left: calc(-10px - var(--gap-words-words)); top: calc(50% - 5px); }
|
|
16036
|
+
.usage-key b { font-weight: 600; color: var(--ink); font-variant-numeric: tabular-nums; }
|
|
16037
|
+
.usage-swatch { width: 10px; height: 10px; background: var(--part-color); }
|
|
15984
16038
|
`,
|
|
15985
16039
|
render: usageMeter,
|
|
15986
16040
|
examples: [
|
|
@@ -15991,7 +16045,30 @@ component({
|
|
|
15991
16045
|
{ label: "A little used: still a sliver", markup: meterPane({ label: "Requests this month", usedNumber: 140, limitNumber: 1e5, unit: "requests", resetDate: "2026-11-01" }) },
|
|
15992
16046
|
{ label: "None used", markup: meterPane({ label: "Database rows", usedNumber: 0, limitNumber: 5e4, unit: "rows" }) },
|
|
15993
16047
|
{ label: "In its own words: a share of a free allowance", markup: meterPane({ label: "Starfall", usedNumber: 62, limitNumber: 100, unit: "%", resetDate: "2026-11-01", shownWords: "62% of this month's free hosting" }) },
|
|
15994
|
-
{ label: "Two allowances on one track: free, then a budget the game lives on (used up is not a warning until the budget is)", markup: meterPane({ label: "Starfall", usedNumber: 100 + 2 * 45, limitNumber: 300, warnFromNumber: 260, isFullFine: true, unit: "", resetDate: "2026-11-01", shownWords: "Free part used. $18 of $40 budget" }) }
|
|
16048
|
+
{ label: "Two allowances on one track: free, then a budget the game lives on (used up is not a warning until the budget is)", markup: meterPane({ label: "Starfall", usedNumber: 100 + 2 * 45, limitNumber: 300, warnFromNumber: 260, isFullFine: true, unit: "", resetDate: "2026-11-01", shownWords: "Free part used. $18 of $40 budget" }) },
|
|
16049
|
+
{ label: "Split by what used it: a color for each part, with its share in the key", markup: meterPane({ label: "Starfall", usedNumber: 62, limitNumber: 100, unit: "%", resetDate: "2026-11-01", shownWords: "62% of this month's free hosting", parts: [{ label: "Rooms", sharePct: 48, partColorIndex: 0 }, { label: "Game server", sharePct: 27, partColorIndex: 1 }, { label: "Database", sharePct: 15, partColorIndex: 3 }, { label: "Requests", sharePct: 10, partColorIndex: 4 }] }) },
|
|
16050
|
+
{ label: "Split, past 80%", markup: meterPane({ label: "Kestrel", usedNumber: 86, limitNumber: 100, unit: "%", resetDate: "2026-11-01", shownWords: "86% of this month's free hosting", parts: [{ label: "Relay", sharePct: 75, partColorIndex: 2 }, { label: "Rooms", sharePct: 25, partColorIndex: 0 }] }) },
|
|
16051
|
+
{ label: "Split, all used", markup: meterPane({ label: "Ashen Crown", usedNumber: 100, limitNumber: 100, unit: "%", resetDate: "2026-11-01", shownWords: "100% of this month's free hosting", parts: [{ label: "Rooms", sharePct: 100, partColorIndex: 0 }] }) }
|
|
16052
|
+
]
|
|
16053
|
+
});
|
|
16054
|
+
component({
|
|
16055
|
+
name: "Usage track",
|
|
16056
|
+
slug: "usage-track",
|
|
16057
|
+
group: "data",
|
|
16058
|
+
summary: "A usage meter's track and its parts' key alone, for a list row or a plate whose own lines say the numbers: a game's hosting in developer settings and on its dashboard.",
|
|
16059
|
+
styles: "",
|
|
16060
|
+
render: usageTrack,
|
|
16061
|
+
examples: [
|
|
16062
|
+
{
|
|
16063
|
+
label: "Split by what used it, its key under the track, in a list row whose lines say the numbers",
|
|
16064
|
+
markup: glassPane({
|
|
16065
|
+
paneWidth: "fit",
|
|
16066
|
+
body: listRows({
|
|
16067
|
+
whenEmpty: html``,
|
|
16068
|
+
rows: [{ title: "Starfall", detail: "62% of this month's free hosting \xB7 Resets Nov 1", meter: { label: "Starfall", usedNumber: 62, limitNumber: 100, unit: "%", parts: [{ label: "Rooms", sharePct: 48, partColorIndex: 0 }, { label: "Game server", sharePct: 27, partColorIndex: 1 }, { label: "Database", sharePct: 15, partColorIndex: 3 }, { label: "Requests", sharePct: 10, partColorIndex: 4 }] } }]
|
|
16069
|
+
})
|
|
16070
|
+
})
|
|
16071
|
+
}
|
|
15995
16072
|
]
|
|
15996
16073
|
});
|
|
15997
16074
|
|
package/bin/node-parts.js
CHANGED
|
@@ -96,7 +96,8 @@ export function localDatabase(file) {
|
|
|
96
96
|
statement.setReadBigInts(true);
|
|
97
97
|
for (const row of statement.iterate(...values)) {
|
|
98
98
|
if (rows.length >= DATABASE_LIMITS.maximumRowCount) throw tooManyRows();
|
|
99
|
-
|
|
99
|
+
// node:sqlite gives an INTEGER as a BigInt and a REAL as a number: a REAL as big as 1e20 is no whole number to refuse.
|
|
100
|
+
rows.push(checkedRow(numbersFrom(row), new Set(Object.entries(row).filter(([, columnValue]) => typeof columnValue === "number").map(([column]) => column))));
|
|
100
101
|
}
|
|
101
102
|
return rows;
|
|
102
103
|
});
|
package/package.json
CHANGED
package/src/database.d.ts
CHANGED
|
@@ -76,18 +76,22 @@ export declare function checkPlayerId(playerId: unknown): string;
|
|
|
76
76
|
/** What erasePlayer says when a table's foreign key refuses the delete. */
|
|
77
77
|
export declare function eraseRefused(tableName: string, error: unknown): DatabaseError;
|
|
78
78
|
export declare const DATABASE_LIMITS: {
|
|
79
|
-
/** One statement's SQL. */
|
|
79
|
+
/** One statement's SQL, in characters. */
|
|
80
80
|
readonly maximumSqlCharacterCount: 100000;
|
|
81
|
+
/** One statement's SQL, in UTF-8 bytes: Cloudflare's SQLite takes 100 KB, so text in other scripts is held to it too. */
|
|
82
|
+
readonly maximumSqlSizeBytes: 100000;
|
|
81
83
|
/** Values for one statement's ?s. */
|
|
82
84
|
readonly maximumValueCount: 100;
|
|
83
|
-
/** One text or blob value
|
|
85
|
+
/** One text or blob value a game server or query binds. */
|
|
84
86
|
readonly maximumValueSizeBytes: number;
|
|
87
|
+
/** One text or blob value in an import, which brings back every row an export wrote: what the database holds in a value. */
|
|
88
|
+
readonly maximumImportValueSizeBytes: number;
|
|
85
89
|
/** Statements in one batch. */
|
|
86
90
|
readonly maximumStatementCount: 100;
|
|
87
91
|
/** Rows one statement may answer; past it, page with LIMIT and OFFSET or a WHERE. */
|
|
88
92
|
readonly maximumRowCount: 10000;
|
|
89
93
|
};
|
|
90
|
-
export type DatabaseErrorCode = "bad_sql" | "bad_values" | "bad_player_id" | "too_many_rows" | "hosting_paused" | "database_full" | "database_unavailable";
|
|
94
|
+
export type DatabaseErrorCode = "bad_sql" | "bad_values" | "bad_player_id" | "too_many_rows" | "hosting_paused" | "database_full" | "database_unavailable" | "integer_too_large";
|
|
91
95
|
/** A call the database refused, with a code, what happened and what to do (like the developer API's errors, docs/NAMING.md). */
|
|
92
96
|
export declare class DatabaseError extends Error {
|
|
93
97
|
readonly code: DatabaseErrorCode;
|
|
@@ -110,12 +114,18 @@ export type DatabaseAnswer<T> = {
|
|
|
110
114
|
export declare function readDatabaseAnswer<T>(answer: DatabaseAnswer<T>): T;
|
|
111
115
|
/** Run work, answering its result or the DatabaseError it threw; any other error is SQLite's (bad_sql). */
|
|
112
116
|
export declare function answerDatabase<T>(work: () => T): DatabaseAnswer<T>;
|
|
117
|
+
/**
|
|
118
|
+
* The PRAGMAs Cloudflare's SQLite runs; it refuses the rest (user_version, journal_mode…) with SQLITE_AUTH, so both
|
|
119
|
+
* runtimes refuse them. defer_foreign_keys is what an import's part runs first, in its transaction.
|
|
120
|
+
*/
|
|
121
|
+
export declare const ALLOWED_PRAGMAS: readonly string[];
|
|
113
122
|
/**
|
|
114
123
|
* Check one statement and its values before it runs: one statement only, no transaction control,
|
|
115
|
-
* values of the kinds SQLite takes. Throws a DatabaseError; answers the
|
|
116
|
-
* trailing ; or comment, which SQLite in a Durable Object takes as an empty
|
|
124
|
+
* nothing Cloudflare's SQLite refuses, values of the kinds SQLite takes. Throws a DatabaseError; answers the
|
|
125
|
+
* statement to run (without a trailing ; or comment, which SQLite in a Durable Object takes as an empty
|
|
126
|
+
* statement) and its values. A text or blob value may be up to maximumValueSizeBytes (an import's: more).
|
|
117
127
|
*/
|
|
118
|
-
export declare function checkStatement(sql: unknown, values: unknown): {
|
|
128
|
+
export declare function checkStatement(sql: unknown, values: unknown, maximumValueSizeBytes?: number): {
|
|
119
129
|
sql: string;
|
|
120
130
|
values: DatabaseValue[];
|
|
121
131
|
};
|
|
@@ -128,9 +138,12 @@ export declare function checkBatch(statements: unknown): {
|
|
|
128
138
|
export declare function plainRow(row: Record<string, unknown>): DatabaseRow;
|
|
129
139
|
/**
|
|
130
140
|
* A row as a game's query gets it: plainRow, refusing a whole number past 2^53, which JavaScript would
|
|
131
|
-
* round (a game would read a different number than the one stored). Dumps and previews keep
|
|
141
|
+
* round (a game would read a different number than the one stored): integer_too_large. Dumps and previews keep
|
|
142
|
+
* plainRow. realColumns names the columns that hold REALs, which can be as big (1e20) and are not refused.
|
|
132
143
|
*/
|
|
133
|
-
export declare function checkedRow(row: Record<string, unknown>): DatabaseRow;
|
|
144
|
+
export declare function checkedRow(row: Record<string, unknown>, realColumns?: ReadonlySet<string>): DatabaseRow;
|
|
145
|
+
/** A whole number past 2^53 in a result, which JavaScript can't hold exactly. */
|
|
146
|
+
export declare function integerTooLarge(column: string): DatabaseError;
|
|
134
147
|
export declare function tooManyRows(): DatabaseError;
|
|
135
148
|
/** One statement in a piece of SQL, as scanStatements finds it. */
|
|
136
149
|
export interface ScannedStatement {
|
package/src/database.js
CHANGED
|
@@ -39,18 +39,22 @@ export function eraseRefused(tableName, error) {
|
|
|
39
39
|
return new DatabaseError("bad_sql", `Erasing the player from ${tableName} failed, so nothing was erased: SQLite: ${sqliteMessage}`, "Give the foreign key ON DELETE CASCADE, or turn deleted_account_erase_enabled off and erase in your own order.");
|
|
40
40
|
}
|
|
41
41
|
export const DATABASE_LIMITS = {
|
|
42
|
-
/** One statement's SQL. */
|
|
42
|
+
/** One statement's SQL, in characters. */
|
|
43
43
|
maximumSqlCharacterCount: 100_000,
|
|
44
|
+
/** One statement's SQL, in UTF-8 bytes: Cloudflare's SQLite takes 100 KB, so text in other scripts is held to it too. */
|
|
45
|
+
maximumSqlSizeBytes: 100_000,
|
|
44
46
|
/** Values for one statement's ?s. */
|
|
45
47
|
maximumValueCount: 100,
|
|
46
|
-
/** One text or blob value
|
|
48
|
+
/** One text or blob value a game server or query binds. */
|
|
47
49
|
maximumValueSizeBytes: 1024 * 1024,
|
|
50
|
+
/** One text or blob value in an import, which brings back every row an export wrote: what the database holds in a value. */
|
|
51
|
+
maximumImportValueSizeBytes: 2 * 1024 * 1024,
|
|
48
52
|
/** Statements in one batch. */
|
|
49
53
|
maximumStatementCount: 100,
|
|
50
54
|
/** Rows one statement may answer; past it, page with LIMIT and OFFSET or a WHERE. */
|
|
51
55
|
maximumRowCount: 10_000,
|
|
52
56
|
};
|
|
53
|
-
const DATABASE_ERROR_CODES = ["bad_sql", "bad_values", "bad_player_id", "too_many_rows", "hosting_paused", "database_full", "database_unavailable"];
|
|
57
|
+
const DATABASE_ERROR_CODES = ["bad_sql", "bad_values", "bad_player_id", "too_many_rows", "hosting_paused", "database_full", "database_unavailable", "integer_too_large"];
|
|
54
58
|
/** A call the database refused, with a code, what happened and what to do (like the developer API's errors, docs/NAMING.md). */
|
|
55
59
|
export class DatabaseError extends Error {
|
|
56
60
|
code;
|
|
@@ -86,16 +90,49 @@ export function answerDatabase(work) {
|
|
|
86
90
|
}
|
|
87
91
|
// Statements that run their own transactions or reach other files. batch() is the transaction.
|
|
88
92
|
const REFUSED_FIRST_WORDS = new Set(["BEGIN", "COMMIT", "END", "ROLLBACK", "SAVEPOINT", "RELEASE", "ATTACH", "DETACH"]);
|
|
93
|
+
/**
|
|
94
|
+
* The PRAGMAs Cloudflare's SQLite runs; it refuses the rest (user_version, journal_mode…) with SQLITE_AUTH, so both
|
|
95
|
+
* runtimes refuse them. defer_foreign_keys is what an import's part runs first, in its transaction.
|
|
96
|
+
*/
|
|
97
|
+
export const ALLOWED_PRAGMAS = ["table_info", "table_xinfo", "table_list", "index_list", "index_info", "index_xinfo", "foreign_key_list", "foreign_key_check", "defer_foreign_keys", "quick_check", "integrity_check", "optimize"];
|
|
98
|
+
/** Functions Cloudflare's SQLite refuses, which would answer in the Node runner. */
|
|
99
|
+
const REFUSED_FUNCTIONS = ["sqlite_version", "sqlite_source_id", "load_extension"];
|
|
100
|
+
/** SQL with its quotes, names and comments blanked, so a word is only found where it's SQL. */
|
|
101
|
+
function sqlWithoutQuotes(sql) {
|
|
102
|
+
return sql.replace(/'(?:[^']|'')*'|"(?:[^"]|"")*"|`[^`]*`|\[[^\]]*\]|--[^\n]*|\/\*[\s\S]*?\*\//g, " ");
|
|
103
|
+
}
|
|
104
|
+
/** What Cloudflare's SQLite refuses and Node's would run: PRAGMAs outside ALLOWED_PRAGMAS, TEMP objects and unauthorized functions. */
|
|
105
|
+
function checkRefusedByCloudflare(sql, leadingWords) {
|
|
106
|
+
const [firstWord = "", secondWord = ""] = leadingWords;
|
|
107
|
+
const bare = sqlWithoutQuotes(sql);
|
|
108
|
+
if (firstWord === "PRAGMA") {
|
|
109
|
+
const pragmaName = /^\s*PRAGMA\s+(?:\w+\s*\.\s*)?(\w+)/i.exec(bare)?.[1]?.toLowerCase() ?? "";
|
|
110
|
+
if (!ALLOWED_PRAGMAS.includes(pragmaName)) {
|
|
111
|
+
throw new DatabaseError("bad_sql", `PRAGMA ${pragmaName || "…"} isn't allowed: Zaisma's database doesn't take it, and neither does your own server's runner, so a query runs the same in both.`, `Allowed: ${ALLOWED_PRAGMAS.map((name) => `PRAGMA ${name}`).join(", ")}. To keep a schema version, use a table of your own, like CREATE TABLE schema_version (version_number INTEGER NOT NULL).`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
if (firstWord === "CREATE" && (secondWord === "TEMP" || secondWord === "TEMPORARY")) {
|
|
115
|
+
throw new DatabaseError("bad_sql", `CREATE ${secondWord} isn't allowed: the database keeps no temporary tables or objects.`, "Make an ordinary table and drop it when done, or use a WITH query for a result you only need inside one statement.");
|
|
116
|
+
}
|
|
117
|
+
const refusedFunction = REFUSED_FUNCTIONS.find((name) => new RegExp(`\\b${name}\\s*\\(`, "i").test(bare));
|
|
118
|
+
if (refusedFunction)
|
|
119
|
+
throw new DatabaseError("bad_sql", `${refusedFunction}() isn't allowed: Zaisma's database doesn't take it, and neither does your own server's runner.`, "Keep what you need to know about the database (a schema version, say) in a table of your own.");
|
|
120
|
+
}
|
|
89
121
|
/**
|
|
90
122
|
* Check one statement and its values before it runs: one statement only, no transaction control,
|
|
91
|
-
* values of the kinds SQLite takes. Throws a DatabaseError; answers the
|
|
92
|
-
* trailing ; or comment, which SQLite in a Durable Object takes as an empty
|
|
123
|
+
* nothing Cloudflare's SQLite refuses, values of the kinds SQLite takes. Throws a DatabaseError; answers the
|
|
124
|
+
* statement to run (without a trailing ; or comment, which SQLite in a Durable Object takes as an empty
|
|
125
|
+
* statement) and its values. A text or blob value may be up to maximumValueSizeBytes (an import's: more).
|
|
93
126
|
*/
|
|
94
|
-
export function checkStatement(sql, values) {
|
|
127
|
+
export function checkStatement(sql, values, maximumValueSizeBytes = DATABASE_LIMITS.maximumValueSizeBytes) {
|
|
95
128
|
if (typeof sql !== "string" || !sql.trim())
|
|
96
129
|
throw new DatabaseError("bad_sql", "The SQL isn't text.", 'Give one SQL statement as text, like "SELECT * FROM scores WHERE player_id = ?".');
|
|
97
130
|
if (sql.length > DATABASE_LIMITS.maximumSqlCharacterCount)
|
|
98
131
|
throw new DatabaseError("bad_sql", `The SQL is ${sql.length} characters.`, `Keep one statement under ${DATABASE_LIMITS.maximumSqlCharacterCount} characters, and pass data as values instead of writing it into the SQL.`);
|
|
132
|
+
// Cloudflare's limit is in bytes: three bytes a character at most, so short statements need no count.
|
|
133
|
+
const sqlSizeBytes = sql.length * 3 <= DATABASE_LIMITS.maximumSqlSizeBytes ? 0 : new TextEncoder().encode(sql).length;
|
|
134
|
+
if (sqlSizeBytes > DATABASE_LIMITS.maximumSqlSizeBytes)
|
|
135
|
+
throw new DatabaseError("bad_sql", `The SQL is ${sqlSizeBytes} bytes.`, `Keep one statement under ${DATABASE_LIMITS.maximumSqlSizeBytes} bytes (UTF-8), and pass data as values instead of writing it into the SQL.`);
|
|
99
136
|
const statements = scanStatements(sql);
|
|
100
137
|
if (statements.length === 0)
|
|
101
138
|
throw new DatabaseError("bad_sql", "The SQL has only comments.", 'Give one SQL statement, like "SELECT * FROM scores WHERE player_id = ?".');
|
|
@@ -104,21 +141,22 @@ export function checkStatement(sql, values) {
|
|
|
104
141
|
const firstWord = statements[0].leadingWords[0] ?? "";
|
|
105
142
|
if (REFUSED_FIRST_WORDS.has(firstWord))
|
|
106
143
|
throw new DatabaseError("bad_sql", `${firstWord} isn't allowed.`, "For a transaction, use a batch (room.database.batch([{ sql, values }, ...]) in a game server, batch_database through the API): its statements all happen or none do.");
|
|
144
|
+
checkRefusedByCloudflare(sql.slice(statements[0].startIndex, statements[0].endIndex), statements[0].leadingWords);
|
|
107
145
|
const given = values === undefined || values === null ? [] : values;
|
|
108
146
|
if (!Array.isArray(given))
|
|
109
147
|
throw new DatabaseError("bad_values", "values isn't a list.", 'Give the values for the ?s as a list, in order, like ["player_8f3k2m", 1200].');
|
|
110
148
|
if (given.length > DATABASE_LIMITS.maximumValueCount)
|
|
111
149
|
throw new DatabaseError("bad_values", `There are ${given.length} values.`, `Give up to ${DATABASE_LIMITS.maximumValueCount} values per statement; split big inserts into several statements in one batch.`);
|
|
112
|
-
return { sql: sql.slice(0, statements[0].endIndex), values: given.map(checkValue) };
|
|
150
|
+
return { sql: sql.slice(0, statements[0].endIndex), values: given.map((givenValue, i) => checkValue(givenValue, i, maximumValueSizeBytes)) };
|
|
113
151
|
}
|
|
114
152
|
/** One value for a ?, the i-th (from 0). */
|
|
115
|
-
function checkValue(givenValue, i) {
|
|
153
|
+
function checkValue(givenValue, i, maximumValueSizeBytes) {
|
|
116
154
|
if (typeof givenValue === "number" && Number.isInteger(givenValue) && !Number.isSafeInteger(givenValue))
|
|
117
155
|
throw new DatabaseError("bad_values", `Value ${i + 1} is a whole number past 2^53, which JavaScript can't hold exactly.`, "Store it as text, like String(value), and read it back as text.");
|
|
118
156
|
if (givenValue === null || typeof givenValue === "string" || givenValue instanceof Uint8Array || (typeof givenValue === "number" && Number.isFinite(givenValue))) {
|
|
119
157
|
const sizeBytes = typeof givenValue === "string" ? new TextEncoder().encode(givenValue).length : givenValue instanceof Uint8Array ? givenValue.byteLength : 0;
|
|
120
|
-
if (sizeBytes >
|
|
121
|
-
throw new DatabaseError("bad_values", `Value ${i + 1} is ${sizeBytes} bytes.`, `Keep each text or blob under ${
|
|
158
|
+
if (sizeBytes > maximumValueSizeBytes)
|
|
159
|
+
throw new DatabaseError("bad_values", `Value ${i + 1} is ${sizeBytes} bytes.`, `Keep each text or blob under ${maximumValueSizeBytes / 1024 / 1024} MB; store big files somewhere else and keep a link here.`);
|
|
122
160
|
return givenValue;
|
|
123
161
|
}
|
|
124
162
|
const kind = typeof givenValue === "boolean" ? "true or false" : typeof givenValue === "number" ? String(givenValue) : Array.isArray(givenValue) ? "a list" : typeof givenValue;
|
|
@@ -150,17 +188,21 @@ export function plainRow(row) {
|
|
|
150
188
|
}
|
|
151
189
|
/**
|
|
152
190
|
* A row as a game's query gets it: plainRow, refusing a whole number past 2^53, which JavaScript would
|
|
153
|
-
* round (a game would read a different number than the one stored). Dumps and previews keep
|
|
191
|
+
* round (a game would read a different number than the one stored): integer_too_large. Dumps and previews keep
|
|
192
|
+
* plainRow. realColumns names the columns that hold REALs, which can be as big (1e20) and are not refused.
|
|
154
193
|
*/
|
|
155
|
-
export function checkedRow(row) {
|
|
194
|
+
export function checkedRow(row, realColumns) {
|
|
156
195
|
const plain = plainRow(row);
|
|
157
196
|
for (const [column, columnValue] of Object.entries(plain)) {
|
|
158
|
-
if (typeof columnValue === "number" && Number.isInteger(columnValue) && !Number.isSafeInteger(columnValue))
|
|
159
|
-
throw
|
|
160
|
-
}
|
|
197
|
+
if (typeof columnValue === "number" && Number.isInteger(columnValue) && !Number.isSafeInteger(columnValue) && !realColumns?.has(column))
|
|
198
|
+
throw integerTooLarge(column);
|
|
161
199
|
}
|
|
162
200
|
return plain;
|
|
163
201
|
}
|
|
202
|
+
/** A whole number past 2^53 in a result, which JavaScript can't hold exactly. */
|
|
203
|
+
export function integerTooLarge(column) {
|
|
204
|
+
return new DatabaseError("integer_too_large", `Column ${column} holds a whole number past 2^53 (9,007,199,254,740,991), which JavaScript can't hold exactly.`, `Read it as text with CAST(${column} AS TEXT), or store 64-bit IDs as TEXT.`);
|
|
205
|
+
}
|
|
164
206
|
export function tooManyRows() {
|
|
165
207
|
return new DatabaseError("too_many_rows", `The statement answered more than ${DATABASE_LIMITS.maximumRowCount} rows.`, "Ask for fewer with LIMIT (and OFFSET or a WHERE on the last row for the next page), or count with SELECT count(*).");
|
|
166
208
|
}
|
package/src/dump.d.ts
CHANGED
|
@@ -54,7 +54,10 @@ export interface ImportPlan {
|
|
|
54
54
|
* statements, and leaves out what sqlite3 writes around them (the transaction, PRAGMAs, ANALYZE and its
|
|
55
55
|
* sqlite_stat1 rows). The values in an INSERT become values for ?s, since Zaisma's database takes
|
|
56
56
|
* statements up to 100 KB and a row of a dump can be bigger; unistr('…'), which sqlite3 writes for text with
|
|
57
|
-
* line breaks, becomes the text it stands for. AUTOINCREMENT's counters only ever go up.
|
|
57
|
+
* line breaks, becomes the text it stands for. AUTOINCREMENT's counters only ever go up. The tables are made
|
|
58
|
+
* first, so a row can refer to a table made after it. A search table in sqlite3's own dump (its
|
|
59
|
+
* `INSERT INTO sqlite_schema` under writable_schema, and its shadow tables) becomes its CREATE VIRTUAL TABLE, filled again
|
|
60
|
+
* from its _content rows with their rowids.
|
|
58
61
|
*/
|
|
59
62
|
export declare function importStatements(dumpSql: string): ImportPlan;
|
|
60
63
|
/** The start of a statement, for an error. */
|
package/src/dump.js
CHANGED
|
@@ -53,8 +53,12 @@ export function dumpPart(read, cursor, partSizeBytes = DUMP_LIMITS.exportPartSiz
|
|
|
53
53
|
};
|
|
54
54
|
let tableIndex = 0;
|
|
55
55
|
let position = null;
|
|
56
|
-
if (cursor === null)
|
|
56
|
+
if (cursor === null) {
|
|
57
57
|
add(DUMP_START);
|
|
58
|
+
// Every table is made before any row goes in, so a row can refer to a table made after it.
|
|
59
|
+
for (const table of tables)
|
|
60
|
+
add(`${table.sql};\n`);
|
|
61
|
+
}
|
|
58
62
|
else {
|
|
59
63
|
position = readCursor(cursor);
|
|
60
64
|
tableIndex = tables.findIndex((table) => table.name === position.table_name);
|
|
@@ -63,29 +67,39 @@ export function dumpPart(read, cursor, partSizeBytes = DUMP_LIMITS.exportPartSiz
|
|
|
63
67
|
}
|
|
64
68
|
for (; tableIndex < tables.length; tableIndex++) {
|
|
65
69
|
const table = tables[tableIndex];
|
|
66
|
-
|
|
67
|
-
add(`${table.sql};\n`);
|
|
68
|
-
position ??= { table_name: table.name, last_rowid_number: table.is_without_rowid ? null : 0, dumped_row_count: 0 };
|
|
70
|
+
position ??= { table_name: table.name, last_rowid_number: null, last_key: null };
|
|
69
71
|
const columns = select("SELECT name, hidden = 0 AS is_insertable, pk AS primary_key_number FROM pragma_table_xinfo(?) ORDER BY cid", [table.name]);
|
|
70
72
|
// Generated columns, and a virtual table's hidden ones, can't be inserted: the INSERTs name the rest.
|
|
71
73
|
const insertedColumns = columns.filter((column) => column.is_insertable === 1);
|
|
72
|
-
|
|
74
|
+
// A search table's rows are known by their rowids, so they go in with them.
|
|
75
|
+
const isVirtual = table.table_kind === "virtual";
|
|
76
|
+
const insertInto = `INSERT INTO ${quoteName(table.name)}${insertedColumns.length < columns.length || isVirtual ? `(${[...(isVirtual ? ["rowid"] : []), ...insertedColumns.map((column) => quoteName(column.name))].join(",")})` : ""} VALUES(`;
|
|
73
77
|
// quote() writes each value as SQL, the way sqlite3's .dump does: 'text' with '' for a quote, X'…' for bytes.
|
|
74
78
|
const valuesSql = insertedColumns.map((column, i) => `quote(${quoteName(column.name)}) AS value_${i}`).join(", ");
|
|
79
|
+
// A table with a declared primary key is read in its key order (rowid last, where it has one), so a row the game deletes or
|
|
80
|
+
// replaces on the other side of the cursor can't move it; a table without one, in rowid order.
|
|
75
81
|
const primaryKey = columns.filter((column) => column.primary_key_number > 0).sort((first, second) => first.primary_key_number - second.primary_key_number);
|
|
82
|
+
const tableName = quoteName(table.name);
|
|
83
|
+
const keyExpressions = primaryKey.length ? [...primaryKey.map((column) => `${tableName}.${quoteName(column.name)}`), ...(table.is_without_rowid ? [] : [`${tableName}.rowid`])] : [];
|
|
76
84
|
while (true) {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
85
|
+
let rows;
|
|
86
|
+
if (keyExpressions.length) {
|
|
87
|
+
const keySql = keyExpressions.map((expression, i) => `typeof(${expression}) AS key_type_${i}, CASE typeof(${expression}) WHEN 'text' THEN ${expression} WHEN 'blob' THEN hex(${expression}) WHEN 'null' THEN NULL ELSE quote(${expression}) END AS key_text_${i}`).join(", ");
|
|
88
|
+
const after = keyAfter(keyExpressions, position.last_key);
|
|
89
|
+
rows = select(`SELECT ${keySql}${valuesSql ? `, ${valuesSql}` : ""} FROM ${tableName}${after.sql ? ` WHERE ${after.sql}` : ""} ORDER BY ${keyExpressions.join(", ")} LIMIT ${PAGE_ROW_COUNT}`, after.values);
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
rows = select(`SELECT rowid AS paging_rowid${valuesSql ? `, ${valuesSql}` : ""} FROM ${tableName} WHERE rowid > ? ORDER BY rowid LIMIT ${PAGE_ROW_COUNT}`, [position.last_rowid_number ?? 0]);
|
|
93
|
+
}
|
|
80
94
|
for (const row of rows) {
|
|
81
|
-
const insert = `${insertInto}${insertedColumns.map((_, i) => row[`value_${i}`]).join(",")});\n`;
|
|
95
|
+
const insert = `${insertInto}${[...(isVirtual ? [row.paging_rowid] : []), ...insertedColumns.map((_, i) => row[`value_${i}`])].join(",")});\n`;
|
|
82
96
|
// The part ends before a row that would take it past partSizeBytes; the next starts with that row.
|
|
83
97
|
if (hasRows && dumpSizeBytes + utf8SizeBytes(insert) > partSizeBytes)
|
|
84
98
|
return { dumpSql, nextCursor: writeCursor(position), readRowCount };
|
|
85
99
|
add(insert);
|
|
86
100
|
hasRows = true;
|
|
87
|
-
if (
|
|
88
|
-
position.
|
|
101
|
+
if (keyExpressions.length)
|
|
102
|
+
position.last_key = keyExpressions.map((_, i) => ({ key_type: row[`key_type_${i}`], key_text: row[`key_text_${i}`] }));
|
|
89
103
|
else
|
|
90
104
|
position.last_rowid_number = row.paging_rowid;
|
|
91
105
|
}
|
|
@@ -131,26 +145,135 @@ function readCursor(cursor) {
|
|
|
131
145
|
catch {
|
|
132
146
|
throw new DumpCursorError("The cursor isn't one an export answered.");
|
|
133
147
|
}
|
|
134
|
-
const { table_name: tableName, last_rowid_number: lastRowidNumber,
|
|
135
|
-
|
|
148
|
+
const { table_name: tableName, last_rowid_number: lastRowidNumber, last_key: lastKey } = (position ?? {});
|
|
149
|
+
const isKey = (key) => {
|
|
150
|
+
const { key_type: keyType, key_text: keyText } = (key ?? {});
|
|
151
|
+
return (keyType === "null" ? keyText === null : (keyType === "integer" || keyType === "real" || keyType === "text" || keyType === "blob") && typeof keyText === "string") && (keyType !== "integer" || /^-?\d+$/.test(keyText)) && (keyType !== "real" || /^-?[0-9.]+(e[+-]?\d+)?$/i.test(keyText)) && (keyType !== "blob" || /^([0-9A-Fa-f]{2})*$/.test(keyText));
|
|
152
|
+
};
|
|
153
|
+
if (typeof tableName !== "string" || !(lastRowidNumber === null || Number.isInteger(lastRowidNumber)) || !(lastKey === null || (Array.isArray(lastKey) && lastKey.length > 0 && lastKey.length <= 64 && lastKey.every(isKey))))
|
|
136
154
|
throw new DumpCursorError("The cursor isn't one an export answered.");
|
|
137
|
-
return { table_name: tableName, last_rowid_number: lastRowidNumber,
|
|
155
|
+
return { table_name: tableName, last_rowid_number: lastRowidNumber, last_key: lastKey };
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* The WHERE for the rows after a key, for ORDER BY the key's columns: one OR per column, each holding the ones before it equal
|
|
159
|
+
* and itself greater (NULLs sort first). The key's values are bound, numbers as text SQLite casts back, so none loses digits.
|
|
160
|
+
*/
|
|
161
|
+
function keyAfter(keyExpressions, lastKey) {
|
|
162
|
+
if (!lastKey)
|
|
163
|
+
return { sql: "", values: [] };
|
|
164
|
+
if (lastKey.length !== keyExpressions.length)
|
|
165
|
+
throw new DumpCursorError("The cursor isn't one an export answered.");
|
|
166
|
+
const values = [];
|
|
167
|
+
const operand = (key) => {
|
|
168
|
+
values.push(key.key_type === "blob" ? Uint8Array.from({ length: key.key_text.length / 2 }, (_, byteIndex) => parseInt(key.key_text.slice(byteIndex * 2, byteIndex * 2 + 2), 16)) : key.key_text);
|
|
169
|
+
return key.key_type === "integer" ? "CAST(? AS INTEGER)" : key.key_type === "real" ? "CAST(? AS REAL)" : "?";
|
|
170
|
+
};
|
|
171
|
+
const terms = keyExpressions.map((expression, i) => {
|
|
172
|
+
const conditions = keyExpressions.slice(0, i).map((earlier, earlierIndex) => (lastKey[earlierIndex].key_type === "null" ? `${earlier} IS NULL` : `${earlier} IS ${operand(lastKey[earlierIndex])}`));
|
|
173
|
+
conditions.push(lastKey[i].key_type === "null" ? `${expression} IS NOT NULL` : `${expression} > ${operand(lastKey[i])}`);
|
|
174
|
+
return `(${conditions.join(" AND ")})`;
|
|
175
|
+
});
|
|
176
|
+
return { sql: terms.join(" OR "), values };
|
|
138
177
|
}
|
|
139
178
|
const IMPORT_FIX = "Import takes a dump's CREATE and INSERT statements, as sqlite3's .dump or export_database writes them; it only adds. Run other changes with query_database or batch_database.";
|
|
179
|
+
/** The name a statement starts with after `CREATE TABLE [IF NOT EXISTS]` or `INSERT INTO`, however it's quoted. */
|
|
180
|
+
function tableNameAfter(text, introduction) {
|
|
181
|
+
const match = new RegExp(`^${introduction.source}\\s*(?:"((?:[^"]|"")*)"|'((?:[^']|'')*)'|\`([^\`]*)\`|\\[([^\\]]*)\\]|(\\w+))`, "i").exec(text);
|
|
182
|
+
if (!match)
|
|
183
|
+
return null;
|
|
184
|
+
const [, doubleQuoted, singleQuoted, backQuoted, bracketed, bare] = match;
|
|
185
|
+
return doubleQuoted?.replaceAll('""', '"') ?? singleQuoted?.replaceAll("''", "'") ?? backQuoted ?? bracketed ?? bare ?? null;
|
|
186
|
+
}
|
|
187
|
+
/** The shadow tables an FTS5 table makes for itself: sqlite3's .dump writes them as ordinary tables, and a search table makes them again. */
|
|
188
|
+
const FTS5_SHADOW_SUFFIXES = ["data", "idx", "content", "docsize", "config"];
|
|
189
|
+
const SEARCH_TABLE_FIX = "Import takes a search table made with fts5. For others, import the database file instead: npx @zaisma/cli database import <slug> game.sqlite.";
|
|
190
|
+
/** The columns of an fts5(…) table from its CREATE VIRTUAL TABLE: the arguments that aren't options (name=value). */
|
|
191
|
+
function searchColumnNames(createSql) {
|
|
192
|
+
const openIndex = createSql.indexOf("(", createSql.search(/fts5/i));
|
|
193
|
+
const closeIndex = createSql.lastIndexOf(")");
|
|
194
|
+
if (openIndex === -1 || closeIndex <= openIndex)
|
|
195
|
+
return [];
|
|
196
|
+
const argumentTexts = [];
|
|
197
|
+
let current = "";
|
|
198
|
+
let openParenthesisCount = 0;
|
|
199
|
+
let quote = null;
|
|
200
|
+
for (const character of createSql.slice(openIndex + 1, closeIndex)) {
|
|
201
|
+
if (quote) {
|
|
202
|
+
current += character;
|
|
203
|
+
if (character === quote)
|
|
204
|
+
quote = null;
|
|
205
|
+
}
|
|
206
|
+
else if (character === "'" || character === '"' || character === "`") {
|
|
207
|
+
quote = character;
|
|
208
|
+
current += character;
|
|
209
|
+
}
|
|
210
|
+
else if (character === "(") {
|
|
211
|
+
openParenthesisCount++;
|
|
212
|
+
current += character;
|
|
213
|
+
}
|
|
214
|
+
else if (character === ")") {
|
|
215
|
+
openParenthesisCount--;
|
|
216
|
+
current += character;
|
|
217
|
+
}
|
|
218
|
+
else if (character === "," && openParenthesisCount === 0) {
|
|
219
|
+
argumentTexts.push(current);
|
|
220
|
+
current = "";
|
|
221
|
+
}
|
|
222
|
+
else
|
|
223
|
+
current += character;
|
|
224
|
+
}
|
|
225
|
+
argumentTexts.push(current);
|
|
226
|
+
const names = [];
|
|
227
|
+
for (const argument of argumentTexts) {
|
|
228
|
+
const trimmed = argument.trim();
|
|
229
|
+
if (!trimmed || /^[\w.]+\s*=/.test(trimmed))
|
|
230
|
+
continue;
|
|
231
|
+
const first = /^(?:"((?:[^"]|"")*)"|'([^']*)'|`([^`]*)`|\[([^\]]*)\]|(\w+))/.exec(trimmed);
|
|
232
|
+
const name = first?.[1]?.replaceAll('""', '"') ?? first?.[2] ?? first?.[3] ?? first?.[4] ?? first?.[5];
|
|
233
|
+
if (name)
|
|
234
|
+
names.push(name);
|
|
235
|
+
}
|
|
236
|
+
return names;
|
|
237
|
+
}
|
|
140
238
|
/**
|
|
141
239
|
* Reads a part of a dump into the statements to run. Import only adds: it takes CREATE and INSERT
|
|
142
240
|
* statements, and leaves out what sqlite3 writes around them (the transaction, PRAGMAs, ANALYZE and its
|
|
143
241
|
* sqlite_stat1 rows). The values in an INSERT become values for ?s, since Zaisma's database takes
|
|
144
242
|
* statements up to 100 KB and a row of a dump can be bigger; unistr('…'), which sqlite3 writes for text with
|
|
145
|
-
* line breaks, becomes the text it stands for. AUTOINCREMENT's counters only ever go up.
|
|
243
|
+
* line breaks, becomes the text it stands for. AUTOINCREMENT's counters only ever go up. The tables are made
|
|
244
|
+
* first, so a row can refer to a table made after it. A search table in sqlite3's own dump (its
|
|
245
|
+
* `INSERT INTO sqlite_schema` under writable_schema, and its shadow tables) becomes its CREATE VIRTUAL TABLE, filled again
|
|
246
|
+
* from its _content rows with their rowids.
|
|
146
247
|
*/
|
|
147
248
|
export function importStatements(dumpSql) {
|
|
148
249
|
const statements = [];
|
|
149
250
|
const scanned = scanStatements(dumpSql);
|
|
150
251
|
let skippedStatementCount = 0;
|
|
252
|
+
const textOf = (scannedStatement) => dumpSql.slice(scannedStatement.startIndex, scannedStatement.endIndex);
|
|
253
|
+
const refuseAt = (statementNumber, text, message, fix = IMPORT_FIX) => new DatabaseError("bad_sql", `Statement ${statementNumber} of the dump (${preview(text)}): ${message}`, fix);
|
|
254
|
+
// sqlite3's .dump creates a virtual table by writing its row in the schema table: those come first, so the shadow tables are known.
|
|
255
|
+
const searchTables = new Map();
|
|
256
|
+
const shadowNames = new Map();
|
|
257
|
+
scanned.forEach((scannedStatement, i) => {
|
|
258
|
+
const [firstWord = "", secondWord = "", thirdWord = ""] = scannedStatement.leadingWords;
|
|
259
|
+
if (!(firstWord === "INSERT" && secondWord === "INTO" && (thirdWord === "SQLITE_SCHEMA" || thirdWord === "SQLITE_MASTER")))
|
|
260
|
+
return;
|
|
261
|
+
const text = textOf(scannedStatement);
|
|
262
|
+
const match = /^INSERT\s+INTO\s+sqlite_(?:schema|master)\s*\([^)]*\)\s*VALUES\s*\(\s*'table'\s*,\s*'((?:[^']|'')*)'\s*,\s*'(?:[^']|'')*'\s*,\s*\d+\s*,\s*'((?:[^']|'')*)'\s*\)$/i.exec(text);
|
|
263
|
+
const createSql = match?.[2]?.replaceAll("''", "'");
|
|
264
|
+
if (!match || !createSql || !/^CREATE\s+VIRTUAL\s+TABLE\s/i.test(createSql))
|
|
265
|
+
throw refuseAt(i + 1, text, "it writes to the schema table, and only a virtual table's can be read from there.");
|
|
266
|
+
if (!/\bUSING\s+fts5\b/i.test(createSql))
|
|
267
|
+
throw refuseAt(i + 1, text, "this virtual table isn't an fts5 search table.", SEARCH_TABLE_FIX);
|
|
268
|
+
const name = match[1].replaceAll("''", "'");
|
|
269
|
+
const table = { name, createSql, columnNames: searchColumnNames(createSql) };
|
|
270
|
+
searchTables.set(i, table);
|
|
271
|
+
for (const suffix of FTS5_SHADOW_SUFFIXES)
|
|
272
|
+
shadowNames.set(`${name}_${suffix}`.toLowerCase(), { ...table, suffix });
|
|
273
|
+
});
|
|
151
274
|
scanned.forEach((scannedStatement, i) => {
|
|
152
275
|
const statementNumber = i + 1;
|
|
153
|
-
const text =
|
|
276
|
+
const text = textOf(scannedStatement);
|
|
154
277
|
const [firstWord = "", secondWord = "", thirdWord = ""] = scannedStatement.leadingWords;
|
|
155
278
|
const words = scannedStatement.leadingWords.join(" ");
|
|
156
279
|
const isSkipped = ["BEGIN", "COMMIT", "END", "ROLLBACK", "ANALYZE"].includes(firstWord) ||
|
|
@@ -162,7 +285,30 @@ export function importStatements(dumpSql) {
|
|
|
162
285
|
skippedStatementCount++;
|
|
163
286
|
return;
|
|
164
287
|
}
|
|
165
|
-
const refuse = (message) =>
|
|
288
|
+
const refuse = (message) => refuseAt(statementNumber, text, message);
|
|
289
|
+
if (firstWord === "INSERT" && secondWord === "INTO" && (thirdWord === "SQLITE_SCHEMA" || thirdWord === "SQLITE_MASTER")) {
|
|
290
|
+
statements.push({ sql: searchTables.get(i).createSql, values: [], statementNumber });
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
if (shadowNames.size) {
|
|
294
|
+
const shadow = shadowNames.get(tableNameAfter(text, firstWord === "CREATE" ? /CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?/ : /INSERT\s+INTO/)?.toLowerCase() ?? "");
|
|
295
|
+
if (shadow && (firstWord === "CREATE" || firstWord === "INSERT")) {
|
|
296
|
+
if (firstWord === "CREATE" || shadow.suffix !== "content") {
|
|
297
|
+
skippedStatementCount++;
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
// The rows of a search table are in its _content, by rowid; they go back in through the table itself.
|
|
301
|
+
const bound = bindLiterals(text, refuse);
|
|
302
|
+
const insertInto = `INSERT INTO ${quoteName(shadow.name)}(${["rowid", ...shadow.columnNames.map(quoteName)].join(",")}) VALUES(`;
|
|
303
|
+
const replaced = bound.sql.replace(/^INSERT\s+INTO\s+(?:"(?:[^"]|"")*"|\w+)\s+VALUES\s*\(/i, insertInto);
|
|
304
|
+
if (replaced === bound.sql)
|
|
305
|
+
throw refuse("a search table's content rows are INSERT INTO \"name_content\" VALUES(…).");
|
|
306
|
+
if (bound.values.length > DATABASE_LIMITS.maximumValueCount)
|
|
307
|
+
throw new DatabaseError("bad_values", `Statement ${statementNumber} of the dump (${preview(text)}) has ${bound.values.length} text and blob values.`, `Give each row its own INSERT, as sqlite3's .dump does: the database takes up to ${DATABASE_LIMITS.maximumValueCount} values a statement.`);
|
|
308
|
+
statements.push({ sql: replaced, values: bound.values, statementNumber });
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
}
|
|
166
312
|
if (firstWord === "CREATE") {
|
|
167
313
|
statements.push({ sql: text, values: [], statementNumber });
|
|
168
314
|
return;
|
|
@@ -188,7 +334,9 @@ export function importStatements(dumpSql) {
|
|
|
188
334
|
throw new DatabaseError("bad_values", `Statement ${statementNumber} of the dump (${preview(text)}) has ${bound.values.length} text and blob values.`, `Give each row its own INSERT, as sqlite3's .dump does: the database takes up to ${DATABASE_LIMITS.maximumValueCount} values a statement.`);
|
|
189
335
|
statements.push({ sql: bound.sql, values: bound.values, statementNumber });
|
|
190
336
|
});
|
|
191
|
-
|
|
337
|
+
// Tables first, in the order given, then the rest in theirs: a row may refer to a table made after it.
|
|
338
|
+
const isTable = (statement) => /^CREATE\s+(?:VIRTUAL\s+)?TABLE\s/i.test(statement.sql);
|
|
339
|
+
return { statements: [...statements.filter(isTable), ...statements.filter((statement) => !isTable(statement))], statementCount: scanned.length, skippedStatementCount };
|
|
192
340
|
}
|
|
193
341
|
/** The start of a statement, for an error. */
|
|
194
342
|
export function preview(text) {
|