@noormdev/sdk 1.0.0-alpha.16 → 1.0.0-alpha.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-BTXUEWYQ.js → chunk-KF7GE2VM.js} +4 -6
- package/dist/chunk-KF7GE2VM.js.map +1 -0
- package/dist/{chunk-ZRZ7AECF.js → chunk-LJDHQEWW.js} +63 -25
- package/dist/chunk-LJDHQEWW.js.map +1 -0
- package/dist/engine-IFYFFWFR.js +4 -0
- package/dist/{engine-5K7FA66A.js.map → engine-IFYFFWFR.js.map} +1 -1
- package/dist/index.d.ts +2502 -2400
- package/dist/index.js +917 -443
- package/dist/index.js.map +1 -1
- package/dist/{storage-FJP4ED7H.js → storage-NDKBEHIZ.js} +3 -3
- package/dist/{storage-FJP4ED7H.js.map → storage-NDKBEHIZ.js.map} +1 -1
- package/package.json +1 -1
- package/dist/chunk-BTXUEWYQ.js.map +0 -1
- package/dist/chunk-ZRZ7AECF.js.map +0 -1
- package/dist/engine-5K7FA66A.js +0 -4
package/dist/index.js
CHANGED
|
@@ -1,19 +1,22 @@
|
|
|
1
|
-
import { processFile, DtReader, isTemplate, FORMAT_VERSION, DT_EXTENSIONS, encryptWithPassphrase, ENCODED_TYPES
|
|
2
|
-
import {
|
|
1
|
+
import { processFile, DtReader, isTemplate, FORMAT_VERSION, DT_EXTENSIONS, encryptWithPassphrase, ENCODED_TYPES } from './chunk-KF7GE2VM.js';
|
|
2
|
+
import { initializeVault, getVaultStatus, setVaultSecret, getVaultSecret, getAllVaultSecrets, listVaultSecretKeys, deleteVaultSecret, vaultSecretExists, getVaultKey, noormDb, getNoormTables, encryptVaultKey } from './chunk-LJDHQEWW.js';
|
|
3
3
|
import { observer } from './chunk-4HW4CS2C.js';
|
|
4
4
|
import { constants, existsSync, readFileSync, mkdirSync, writeFileSync, createWriteStream } from 'fs';
|
|
5
|
-
import path, { join, dirname } from 'path';
|
|
5
|
+
import path, { join, dirname, resolve } from 'path';
|
|
6
6
|
import { makeNestedConfig, gigabytes, attempt, retry, clone, merge, attemptSync } from '@logosdx/utils';
|
|
7
|
-
import { homedir, userInfo } from 'os';
|
|
7
|
+
import { homedir, availableParallelism, userInfo } from 'os';
|
|
8
8
|
import { stat, mkdir, writeFile, unlink, rename, rm, readdir, readFile, access } from 'fs/promises';
|
|
9
|
-
import { createHash, createDecipheriv, randomBytes, createCipheriv, hkdfSync } from 'crypto';
|
|
9
|
+
import { createHash, createDecipheriv, randomBytes, createCipheriv, randomUUID, hkdfSync } from 'crypto';
|
|
10
10
|
import { z } from 'zod';
|
|
11
11
|
import { parse, stringify } from 'yaml';
|
|
12
12
|
import { execSync } from 'child_process';
|
|
13
13
|
import { sql } from 'kysely';
|
|
14
14
|
import 'dayjs';
|
|
15
15
|
import ansis from 'ansis';
|
|
16
|
-
import {
|
|
16
|
+
import { Worker, parentPort, workerData } from 'worker_threads';
|
|
17
|
+
import { ObserverRelay } from '@logosdx/observer';
|
|
18
|
+
import { fileURLToPath } from 'url';
|
|
19
|
+
import { createGzip } from 'zlib';
|
|
17
20
|
import { pipeline } from 'stream/promises';
|
|
18
21
|
import { PassThrough } from 'stream';
|
|
19
22
|
import JSON5 from 'json5';
|
|
@@ -1672,6 +1675,7 @@ async function loadExistingIdentity() {
|
|
|
1672
1675
|
var ConnectionManager = class {
|
|
1673
1676
|
#cached = /* @__PURE__ */ new Map();
|
|
1674
1677
|
#tracked = /* @__PURE__ */ new Map();
|
|
1678
|
+
#bridges = /* @__PURE__ */ new Map();
|
|
1675
1679
|
#nextId = 1;
|
|
1676
1680
|
#shuttingDown = false;
|
|
1677
1681
|
#unsubscribe = null;
|
|
@@ -1755,7 +1759,7 @@ var ConnectionManager = class {
|
|
|
1755
1759
|
for (const [id, entry] of trackedEntries) {
|
|
1756
1760
|
const destroyWithTimeout = Promise.race([
|
|
1757
1761
|
entry.conn.destroy(),
|
|
1758
|
-
new Promise((
|
|
1762
|
+
new Promise((resolve2) => setTimeout(resolve2, CLOSE_TIMEOUT))
|
|
1759
1763
|
]);
|
|
1760
1764
|
const [, err] = await attempt(() => destroyWithTimeout);
|
|
1761
1765
|
this.#tracked.delete(id);
|
|
@@ -1765,6 +1769,28 @@ var ConnectionManager = class {
|
|
|
1765
1769
|
observer.emit("connection:close", { configName: entry.configName });
|
|
1766
1770
|
}
|
|
1767
1771
|
}
|
|
1772
|
+
const bridgeNames = Array.from(this.#bridges.keys());
|
|
1773
|
+
for (const name of bridgeNames) {
|
|
1774
|
+
await this.closeBridge(name);
|
|
1775
|
+
}
|
|
1776
|
+
}
|
|
1777
|
+
/**
|
|
1778
|
+
* Track a WorkerBridge connection for a config.
|
|
1779
|
+
*
|
|
1780
|
+
* The bridge is shut down alongside regular connections on app:shutdown.
|
|
1781
|
+
*/
|
|
1782
|
+
trackBridge(configName, bridge) {
|
|
1783
|
+
this.#bridges.set(configName, bridge);
|
|
1784
|
+
}
|
|
1785
|
+
/**
|
|
1786
|
+
* Close a specific bridge by config name.
|
|
1787
|
+
*/
|
|
1788
|
+
async closeBridge(configName) {
|
|
1789
|
+
const bridge = this.#bridges.get(configName);
|
|
1790
|
+
if (bridge) {
|
|
1791
|
+
await bridge.shutdown();
|
|
1792
|
+
this.#bridges.delete(configName);
|
|
1793
|
+
}
|
|
1768
1794
|
}
|
|
1769
1795
|
/**
|
|
1770
1796
|
* Check if a cached connection exists for a config name.
|
|
@@ -1776,7 +1802,7 @@ var ConnectionManager = class {
|
|
|
1776
1802
|
* Get total number of active connections.
|
|
1777
1803
|
*/
|
|
1778
1804
|
get size() {
|
|
1779
|
-
return this.#cached.size + this.#tracked.size;
|
|
1805
|
+
return this.#cached.size + this.#tracked.size + this.#bridges.size;
|
|
1780
1806
|
}
|
|
1781
1807
|
/**
|
|
1782
1808
|
* Check if shutdown is in progress.
|
|
@@ -1889,7 +1915,7 @@ async function createConnection(config, configName = "__default__") {
|
|
|
1889
1915
|
destroy: wrappedDestroy
|
|
1890
1916
|
};
|
|
1891
1917
|
observer.emit("connection:open", { configName, dialect: config.dialect });
|
|
1892
|
-
await waitForIdentityToLoad(trackedConn.db);
|
|
1918
|
+
await waitForIdentityToLoad(trackedConn.db, config.dialect);
|
|
1893
1919
|
return trackedConn;
|
|
1894
1920
|
}
|
|
1895
1921
|
var SYSTEM_DATABASES = {
|
|
@@ -1920,7 +1946,7 @@ async function testConnection(config, options = {}) {
|
|
|
1920
1946
|
// src/core/version/types.ts
|
|
1921
1947
|
var CURRENT_VERSIONS = Object.freeze({
|
|
1922
1948
|
/** Database tracking tables schema version */
|
|
1923
|
-
schema:
|
|
1949
|
+
schema: 2,
|
|
1924
1950
|
/** State file (state.enc) schema version */
|
|
1925
1951
|
state: 1,
|
|
1926
1952
|
/** Settings file (settings.yml) schema version */
|
|
@@ -2022,6 +2048,171 @@ var v1 = {
|
|
|
2022
2048
|
await db.schema.dropTable("__noorm_version__").execute();
|
|
2023
2049
|
}
|
|
2024
2050
|
};
|
|
2051
|
+
var TABLE_MAP = [
|
|
2052
|
+
{ old: "__noorm_version__", new: "version" },
|
|
2053
|
+
{ old: "__noorm_change__", new: "change" },
|
|
2054
|
+
{ old: "__noorm_executions__", new: "executions" },
|
|
2055
|
+
{ old: "__noorm_lock__", new: "lock" },
|
|
2056
|
+
{ old: "__noorm_identities__", new: "identities" },
|
|
2057
|
+
{ old: "__noorm_vault__", new: "vault" }
|
|
2058
|
+
];
|
|
2059
|
+
var INDEXES = [
|
|
2060
|
+
{ name: "idx_executions_change_id", table: "executions", columns: ["change_id"] },
|
|
2061
|
+
{ name: "idx_change_name_config", table: "change", columns: ["name", "config_name"] },
|
|
2062
|
+
{ name: "idx_vault_secret_key", table: "vault", columns: ["secret_key"] }
|
|
2063
|
+
];
|
|
2064
|
+
var v2 = {
|
|
2065
|
+
version: 2,
|
|
2066
|
+
description: "Move tracking tables to noorm schema",
|
|
2067
|
+
async up(db, dialect) {
|
|
2068
|
+
if (dialect === "sqlite" || dialect === "mysql") return;
|
|
2069
|
+
await db.transaction().execute(async (trx) => {
|
|
2070
|
+
if (dialect === "postgres") {
|
|
2071
|
+
await sql`CREATE SCHEMA IF NOT EXISTS noorm`.execute(trx);
|
|
2072
|
+
} else {
|
|
2073
|
+
await sql`IF NOT EXISTS (SELECT 1 FROM sys.schemas WHERE name = 'noorm') EXEC('CREATE SCHEMA noorm')`.execute(trx);
|
|
2074
|
+
}
|
|
2075
|
+
if (dialect === "postgres") {
|
|
2076
|
+
const { rows } = await sql`
|
|
2077
|
+
SELECT constraint_name
|
|
2078
|
+
FROM information_schema.table_constraints
|
|
2079
|
+
WHERE table_name = '__noorm_executions__'
|
|
2080
|
+
AND constraint_type = 'FOREIGN KEY'
|
|
2081
|
+
`.execute(trx);
|
|
2082
|
+
for (const row of rows) {
|
|
2083
|
+
await sql`ALTER TABLE __noorm_executions__ DROP CONSTRAINT ${sql.ref(row.constraint_name)}`.execute(trx);
|
|
2084
|
+
}
|
|
2085
|
+
} else {
|
|
2086
|
+
const { rows } = await sql`
|
|
2087
|
+
SELECT fk.name AS fk_name
|
|
2088
|
+
FROM sys.foreign_keys fk
|
|
2089
|
+
JOIN sys.tables t ON fk.parent_object_id = t.object_id
|
|
2090
|
+
WHERE t.name = '__noorm_executions__'
|
|
2091
|
+
`.execute(trx);
|
|
2092
|
+
for (const row of rows) {
|
|
2093
|
+
await sql`ALTER TABLE __noorm_executions__ DROP CONSTRAINT ${sql.ref(row.fk_name)}`.execute(trx);
|
|
2094
|
+
}
|
|
2095
|
+
}
|
|
2096
|
+
if (dialect === "postgres") {
|
|
2097
|
+
for (const idx of INDEXES) {
|
|
2098
|
+
await sql`DROP INDEX IF EXISTS ${sql.ref(idx.name)}`.execute(trx);
|
|
2099
|
+
}
|
|
2100
|
+
} else {
|
|
2101
|
+
const prefixedTables = {
|
|
2102
|
+
executions: "__noorm_executions__",
|
|
2103
|
+
change: "__noorm_change__",
|
|
2104
|
+
vault: "__noorm_vault__"
|
|
2105
|
+
};
|
|
2106
|
+
for (const idx of INDEXES) {
|
|
2107
|
+
const tableName = prefixedTables[idx.table];
|
|
2108
|
+
await sql`
|
|
2109
|
+
IF EXISTS (SELECT 1 FROM sys.indexes WHERE name = ${idx.name})
|
|
2110
|
+
DROP INDEX ${sql.ref(idx.name)} ON ${sql.table(tableName)}
|
|
2111
|
+
`.execute(trx);
|
|
2112
|
+
}
|
|
2113
|
+
}
|
|
2114
|
+
for (const t of TABLE_MAP) {
|
|
2115
|
+
if (dialect === "postgres") {
|
|
2116
|
+
await sql`ALTER TABLE ${sql.table(t.old)} SET SCHEMA noorm`.execute(trx);
|
|
2117
|
+
} else {
|
|
2118
|
+
await sql`ALTER SCHEMA noorm TRANSFER ${sql.ref(`dbo.${t.old}`)}`.execute(trx);
|
|
2119
|
+
}
|
|
2120
|
+
}
|
|
2121
|
+
for (const t of TABLE_MAP) {
|
|
2122
|
+
if (dialect === "postgres") {
|
|
2123
|
+
await sql`ALTER TABLE ${sql.table(`noorm.${t.old}`)} RENAME TO ${sql.ref(t.new)}`.execute(trx);
|
|
2124
|
+
} else {
|
|
2125
|
+
await sql`EXEC sp_rename ${sql.raw(`'noorm.${t.old}'`)}, ${sql.raw(`'${t.new}'`)}`.execute(trx);
|
|
2126
|
+
}
|
|
2127
|
+
}
|
|
2128
|
+
await sql`
|
|
2129
|
+
ALTER TABLE ${sql.table("noorm.executions")}
|
|
2130
|
+
ADD CONSTRAINT fk_executions_change_id
|
|
2131
|
+
FOREIGN KEY (change_id) REFERENCES ${sql.table("noorm.change")}(id)
|
|
2132
|
+
ON DELETE CASCADE
|
|
2133
|
+
`.execute(trx);
|
|
2134
|
+
for (const idx of INDEXES) {
|
|
2135
|
+
const cols = idx.columns.join(", ");
|
|
2136
|
+
await sql`CREATE INDEX ${sql.ref(idx.name)} ON ${sql.table(`noorm.${idx.table}`)} (${sql.raw(cols)})`.execute(trx);
|
|
2137
|
+
}
|
|
2138
|
+
});
|
|
2139
|
+
},
|
|
2140
|
+
async down(db, dialect) {
|
|
2141
|
+
if (dialect === "sqlite" || dialect === "mysql") return;
|
|
2142
|
+
await db.transaction().execute(async (trx) => {
|
|
2143
|
+
if (dialect === "postgres") {
|
|
2144
|
+
const { rows } = await sql`
|
|
2145
|
+
SELECT constraint_name
|
|
2146
|
+
FROM information_schema.table_constraints
|
|
2147
|
+
WHERE table_schema = 'noorm'
|
|
2148
|
+
AND table_name = 'executions'
|
|
2149
|
+
AND constraint_type = 'FOREIGN KEY'
|
|
2150
|
+
`.execute(trx);
|
|
2151
|
+
for (const row of rows) {
|
|
2152
|
+
await sql`ALTER TABLE ${sql.table("noorm.executions")} DROP CONSTRAINT ${sql.ref(row.constraint_name)}`.execute(trx);
|
|
2153
|
+
}
|
|
2154
|
+
} else {
|
|
2155
|
+
const { rows } = await sql`
|
|
2156
|
+
SELECT fk.name AS fk_name
|
|
2157
|
+
FROM sys.foreign_keys fk
|
|
2158
|
+
JOIN sys.tables t ON fk.parent_object_id = t.object_id
|
|
2159
|
+
JOIN sys.schemas s ON t.schema_id = s.schema_id
|
|
2160
|
+
WHERE t.name = 'executions' AND s.name = 'noorm'
|
|
2161
|
+
`.execute(trx);
|
|
2162
|
+
for (const row of rows) {
|
|
2163
|
+
await sql`ALTER TABLE ${sql.table("noorm.executions")} DROP CONSTRAINT ${sql.ref(row.fk_name)}`.execute(trx);
|
|
2164
|
+
}
|
|
2165
|
+
}
|
|
2166
|
+
if (dialect === "postgres") {
|
|
2167
|
+
for (const idx of INDEXES) {
|
|
2168
|
+
await sql`DROP INDEX IF EXISTS ${sql.ref(`noorm.${idx.name}`)}`.execute(trx);
|
|
2169
|
+
}
|
|
2170
|
+
} else {
|
|
2171
|
+
for (const idx of INDEXES) {
|
|
2172
|
+
await sql`
|
|
2173
|
+
IF EXISTS (SELECT 1 FROM sys.indexes WHERE name = ${idx.name})
|
|
2174
|
+
DROP INDEX ${sql.ref(idx.name)} ON ${sql.table(`noorm.${idx.table}`)}
|
|
2175
|
+
`.execute(trx);
|
|
2176
|
+
}
|
|
2177
|
+
}
|
|
2178
|
+
for (const t of TABLE_MAP) {
|
|
2179
|
+
if (dialect === "postgres") {
|
|
2180
|
+
await sql`ALTER TABLE ${sql.table(`noorm.${t.new}`)} RENAME TO ${sql.ref(t.old)}`.execute(trx);
|
|
2181
|
+
} else {
|
|
2182
|
+
await sql`EXEC sp_rename ${sql.raw(`'noorm.${t.new}'`)}, ${sql.raw(`'${t.old}'`)}`.execute(trx);
|
|
2183
|
+
}
|
|
2184
|
+
}
|
|
2185
|
+
for (const t of TABLE_MAP) {
|
|
2186
|
+
if (dialect === "postgres") {
|
|
2187
|
+
await sql`ALTER TABLE ${sql.table(`noorm.${t.old}`)} SET SCHEMA public`.execute(trx);
|
|
2188
|
+
} else {
|
|
2189
|
+
await sql`ALTER SCHEMA dbo TRANSFER ${sql.ref(`noorm.${t.old}`)}`.execute(trx);
|
|
2190
|
+
}
|
|
2191
|
+
}
|
|
2192
|
+
await sql`
|
|
2193
|
+
ALTER TABLE __noorm_executions__
|
|
2194
|
+
ADD CONSTRAINT fk_executions_change_id
|
|
2195
|
+
FOREIGN KEY (change_id) REFERENCES __noorm_change__(id)
|
|
2196
|
+
ON DELETE CASCADE
|
|
2197
|
+
`.execute(trx);
|
|
2198
|
+
for (const idx of INDEXES) {
|
|
2199
|
+
const prefixedTables = {
|
|
2200
|
+
executions: "__noorm_executions__",
|
|
2201
|
+
change: "__noorm_change__",
|
|
2202
|
+
vault: "__noorm_vault__"
|
|
2203
|
+
};
|
|
2204
|
+
const tableName = prefixedTables[idx.table];
|
|
2205
|
+
const cols = idx.columns.join(", ");
|
|
2206
|
+
await sql`CREATE INDEX ${sql.ref(idx.name)} ON ${sql.table(tableName)} (${sql.raw(cols)})`.execute(trx);
|
|
2207
|
+
}
|
|
2208
|
+
if (dialect === "postgres") {
|
|
2209
|
+
await sql`DROP SCHEMA IF EXISTS noorm`.execute(trx);
|
|
2210
|
+
} else {
|
|
2211
|
+
await sql`IF EXISTS (SELECT 1 FROM sys.schemas WHERE name = 'noorm') DROP SCHEMA noorm`.execute(trx);
|
|
2212
|
+
}
|
|
2213
|
+
});
|
|
2214
|
+
}
|
|
2215
|
+
};
|
|
2025
2216
|
|
|
2026
2217
|
// src/core/update/checker.ts
|
|
2027
2218
|
function getCurrentVersion() {
|
|
@@ -2029,26 +2220,43 @@ function getCurrentVersion() {
|
|
|
2029
2220
|
}
|
|
2030
2221
|
|
|
2031
2222
|
// src/core/version/schema/index.ts
|
|
2032
|
-
var MIGRATIONS = [v1];
|
|
2033
|
-
async function tablesExist(db) {
|
|
2034
|
-
const [
|
|
2035
|
-
await
|
|
2223
|
+
var MIGRATIONS = [v1, v2];
|
|
2224
|
+
async function tablesExist(db, dialect) {
|
|
2225
|
+
const [legacyResult] = await attempt(async () => {
|
|
2226
|
+
await db.selectFrom("__noorm_version__").select("id").limit(1).executeTakeFirst();
|
|
2036
2227
|
return true;
|
|
2037
2228
|
});
|
|
2038
|
-
if (
|
|
2039
|
-
|
|
2229
|
+
if (legacyResult) return true;
|
|
2230
|
+
if (dialect === "postgres" || dialect === "mssql") {
|
|
2231
|
+
const ndb = noormDb(db, dialect);
|
|
2232
|
+
const tables = getNoormTables(dialect);
|
|
2233
|
+
const [schemaResult] = await attempt(async () => {
|
|
2234
|
+
await ndb.selectFrom(tables.version).select("id").limit(1).executeTakeFirst();
|
|
2235
|
+
return true;
|
|
2236
|
+
});
|
|
2237
|
+
if (schemaResult) return true;
|
|
2238
|
+
}
|
|
2239
|
+
return false;
|
|
2040
2240
|
}
|
|
2041
|
-
async function getSchemaVersion(db) {
|
|
2042
|
-
const exists = await tablesExist(db);
|
|
2241
|
+
async function getSchemaVersion(db, dialect) {
|
|
2242
|
+
const exists = await tablesExist(db, dialect);
|
|
2043
2243
|
if (!exists) return 0;
|
|
2044
|
-
const [
|
|
2244
|
+
const [legacyResult] = await attempt(async () => {
|
|
2045
2245
|
return db.selectFrom("__noorm_version__").select("noorm_version").orderBy("id", "desc").limit(1).executeTakeFirst();
|
|
2046
2246
|
});
|
|
2047
|
-
if (
|
|
2048
|
-
|
|
2247
|
+
if (legacyResult) return legacyResult.noorm_version ?? 0;
|
|
2248
|
+
if (dialect === "postgres" || dialect === "mssql") {
|
|
2249
|
+
const ndb = noormDb(db, dialect);
|
|
2250
|
+
const tables = getNoormTables(dialect);
|
|
2251
|
+
const [schemaResult] = await attempt(async () => {
|
|
2252
|
+
return ndb.selectFrom(tables.version).select("noorm_version").orderBy("id", "desc").limit(1).executeTakeFirst();
|
|
2253
|
+
});
|
|
2254
|
+
if (schemaResult) return schemaResult.noorm_version ?? 0;
|
|
2255
|
+
}
|
|
2256
|
+
return 0;
|
|
2049
2257
|
}
|
|
2050
|
-
async function checkSchemaVersion(db) {
|
|
2051
|
-
const current = await getSchemaVersion(db);
|
|
2258
|
+
async function checkSchemaVersion(db, dialect) {
|
|
2259
|
+
const current = await getSchemaVersion(db, dialect);
|
|
2052
2260
|
const expected = CURRENT_VERSIONS.schema;
|
|
2053
2261
|
observer.emit("version:schema:checking", { current });
|
|
2054
2262
|
return {
|
|
@@ -2070,7 +2278,9 @@ async function bootstrapSchema(db, dialect, options) {
|
|
|
2070
2278
|
throw new MigrationError("schema", migration.version, err);
|
|
2071
2279
|
}
|
|
2072
2280
|
}
|
|
2073
|
-
|
|
2281
|
+
const ndb = noormDb(db, dialect);
|
|
2282
|
+
const tables = getNoormTables(dialect);
|
|
2283
|
+
await ndb.insertInto(tables.version).values({
|
|
2074
2284
|
cli_version: getCurrentVersion(),
|
|
2075
2285
|
noorm_version: CURRENT_VERSIONS.schema,
|
|
2076
2286
|
state_version: CURRENT_VERSIONS.state,
|
|
@@ -2082,22 +2292,37 @@ async function bootstrapSchema(db, dialect, options) {
|
|
|
2082
2292
|
to: CURRENT_VERSIONS.schema,
|
|
2083
2293
|
durationMs
|
|
2084
2294
|
});
|
|
2085
|
-
await waitForIdentityToLoad(db);
|
|
2295
|
+
await waitForIdentityToLoad(db, dialect);
|
|
2086
2296
|
}
|
|
2087
|
-
async function getLatestVersionRecord(db) {
|
|
2088
|
-
const exists = await tablesExist(db);
|
|
2297
|
+
async function getLatestVersionRecord(db, dialect) {
|
|
2298
|
+
const exists = await tablesExist(db, dialect);
|
|
2089
2299
|
if (!exists) return null;
|
|
2090
|
-
const [
|
|
2300
|
+
const [legacyResult] = await attempt(async () => {
|
|
2091
2301
|
return db.selectFrom("__noorm_version__").select(["state_version", "settings_version"]).orderBy("id", "desc").limit(1).executeTakeFirst();
|
|
2092
2302
|
});
|
|
2093
|
-
if (
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2303
|
+
if (legacyResult) {
|
|
2304
|
+
return {
|
|
2305
|
+
stateVersion: legacyResult.state_version,
|
|
2306
|
+
settingsVersion: legacyResult.settings_version
|
|
2307
|
+
};
|
|
2308
|
+
}
|
|
2309
|
+
if (dialect === "postgres" || dialect === "mssql") {
|
|
2310
|
+
const ndb = noormDb(db, dialect);
|
|
2311
|
+
const tables = getNoormTables(dialect);
|
|
2312
|
+
const [schemaResult] = await attempt(async () => {
|
|
2313
|
+
return ndb.selectFrom(tables.version).select(["state_version", "settings_version"]).orderBy("id", "desc").limit(1).executeTakeFirst();
|
|
2314
|
+
});
|
|
2315
|
+
if (schemaResult) {
|
|
2316
|
+
return {
|
|
2317
|
+
stateVersion: schemaResult.state_version,
|
|
2318
|
+
settingsVersion: schemaResult.settings_version
|
|
2319
|
+
};
|
|
2320
|
+
}
|
|
2321
|
+
}
|
|
2322
|
+
return null;
|
|
2098
2323
|
}
|
|
2099
2324
|
async function migrateSchema(db, dialect, options) {
|
|
2100
|
-
const status = await checkSchemaVersion(db);
|
|
2325
|
+
const status = await checkSchemaVersion(db, dialect);
|
|
2101
2326
|
if (status.isNewer) {
|
|
2102
2327
|
observer.emit("version:mismatch", {
|
|
2103
2328
|
layer: "schema",
|
|
@@ -2116,7 +2341,7 @@ async function migrateSchema(db, dialect, options) {
|
|
|
2116
2341
|
from: status.current,
|
|
2117
2342
|
to: CURRENT_VERSIONS.schema
|
|
2118
2343
|
});
|
|
2119
|
-
const existing = await getLatestVersionRecord(db);
|
|
2344
|
+
const existing = await getLatestVersionRecord(db, dialect);
|
|
2120
2345
|
const pendingMigrations = MIGRATIONS.filter((m) => m.version > status.current);
|
|
2121
2346
|
for (const migration of pendingMigrations) {
|
|
2122
2347
|
const [, err] = await attempt(() => migration.up(db, dialect));
|
|
@@ -2124,7 +2349,9 @@ async function migrateSchema(db, dialect, options) {
|
|
|
2124
2349
|
throw new MigrationError("schema", migration.version, err);
|
|
2125
2350
|
}
|
|
2126
2351
|
}
|
|
2127
|
-
|
|
2352
|
+
const ndb = noormDb(db, dialect);
|
|
2353
|
+
const tables = getNoormTables(dialect);
|
|
2354
|
+
await ndb.insertInto(tables.version).values({
|
|
2128
2355
|
cli_version: getCurrentVersion(),
|
|
2129
2356
|
noorm_version: CURRENT_VERSIONS.schema,
|
|
2130
2357
|
state_version: existing?.stateVersion ?? CURRENT_VERSIONS.state,
|
|
@@ -2136,7 +2363,7 @@ async function migrateSchema(db, dialect, options) {
|
|
|
2136
2363
|
to: CURRENT_VERSIONS.schema,
|
|
2137
2364
|
durationMs
|
|
2138
2365
|
});
|
|
2139
|
-
await waitForIdentityToLoad(db);
|
|
2366
|
+
await waitForIdentityToLoad(db, dialect);
|
|
2140
2367
|
}
|
|
2141
2368
|
async function ensureSchemaVersion(db, dialect, options) {
|
|
2142
2369
|
await migrateSchema(db, dialect);
|
|
@@ -2284,9 +2511,11 @@ addMaskedFields([
|
|
|
2284
2511
|
]);
|
|
2285
2512
|
|
|
2286
2513
|
// src/core/identity/sync.ts
|
|
2287
|
-
async function registerIdentity(db, identity) {
|
|
2514
|
+
async function registerIdentity(db, identity, dialect) {
|
|
2515
|
+
const ndb = noormDb(db, dialect);
|
|
2516
|
+
const tables = getNoormTables(dialect);
|
|
2288
2517
|
const [existing, selectErr] = await attempt(
|
|
2289
|
-
() =>
|
|
2518
|
+
() => ndb.selectFrom(tables.identities).select(["id"]).where("identity_hash", "=", identity.identityHash).executeTakeFirst()
|
|
2290
2519
|
);
|
|
2291
2520
|
if (selectErr) {
|
|
2292
2521
|
observer.emit("error", {
|
|
@@ -2302,7 +2531,7 @@ async function registerIdentity(db, identity) {
|
|
|
2302
2531
|
}
|
|
2303
2532
|
if (existing) {
|
|
2304
2533
|
const [, updateErr] = await attempt(
|
|
2305
|
-
() =>
|
|
2534
|
+
() => ndb.updateTable(tables.identities).set({ last_seen_at: /* @__PURE__ */ new Date() }).where("identity_hash", "=", identity.identityHash).execute()
|
|
2306
2535
|
);
|
|
2307
2536
|
if (updateErr) {
|
|
2308
2537
|
observer.emit("error", {
|
|
@@ -2319,7 +2548,7 @@ async function registerIdentity(db, identity) {
|
|
|
2319
2548
|
return { ok: true, registered: false };
|
|
2320
2549
|
}
|
|
2321
2550
|
const [, insertErr] = await attempt(
|
|
2322
|
-
() =>
|
|
2551
|
+
() => ndb.insertInto(tables.identities).values({
|
|
2323
2552
|
identity_hash: identity.identityHash,
|
|
2324
2553
|
email: identity.email,
|
|
2325
2554
|
name: identity.name,
|
|
@@ -2362,11 +2591,11 @@ function resolveIdentity2(options = {}) {
|
|
|
2362
2591
|
function getIdentityForConfig(config) {
|
|
2363
2592
|
return resolveIdentity2({ configIdentity: config.identity });
|
|
2364
2593
|
}
|
|
2365
|
-
async function waitForIdentityToLoad(db) {
|
|
2594
|
+
async function waitForIdentityToLoad(db, dialect) {
|
|
2366
2595
|
const identity = await loadExistingIdentity();
|
|
2367
2596
|
if (!identity) observer.emit("identity:not-found");
|
|
2368
2597
|
if (!identity) return;
|
|
2369
|
-
const [, err] = await attempt(() => registerIdentity(db, identity));
|
|
2598
|
+
const [, err] = await attempt(() => registerIdentity(db, identity, dialect));
|
|
2370
2599
|
if (err) observer.emit("error", {
|
|
2371
2600
|
error: err,
|
|
2372
2601
|
source: "identity:ensure",
|
|
@@ -2946,9 +3175,13 @@ function getExtension(filename) {
|
|
|
2946
3175
|
}
|
|
2947
3176
|
var ChangeHistory = class {
|
|
2948
3177
|
#db;
|
|
3178
|
+
#ndb;
|
|
3179
|
+
#tables;
|
|
2949
3180
|
#configName;
|
|
2950
|
-
constructor(db, configName) {
|
|
3181
|
+
constructor(db, configName, dialect = "sqlite") {
|
|
2951
3182
|
this.#db = db;
|
|
3183
|
+
this.#ndb = noormDb(db, dialect);
|
|
3184
|
+
this.#tables = getNoormTables(dialect);
|
|
2952
3185
|
this.#configName = configName;
|
|
2953
3186
|
}
|
|
2954
3187
|
// ─────────────────────────────────────────────────────────
|
|
@@ -2965,7 +3198,7 @@ var ChangeHistory = class {
|
|
|
2965
3198
|
*/
|
|
2966
3199
|
async getStatus(name) {
|
|
2967
3200
|
const [record, err] = await attempt(
|
|
2968
|
-
() => this.#
|
|
3201
|
+
() => this.#ndb.selectFrom(this.#tables.change).select([
|
|
2969
3202
|
"name",
|
|
2970
3203
|
"status",
|
|
2971
3204
|
"executed_at",
|
|
@@ -2986,7 +3219,7 @@ var ChangeHistory = class {
|
|
|
2986
3219
|
return null;
|
|
2987
3220
|
}
|
|
2988
3221
|
const [revertRecord] = await attempt(
|
|
2989
|
-
() => this.#
|
|
3222
|
+
() => this.#ndb.selectFrom(this.#tables.change).select(["executed_at"]).where("name", "=", name).where("change_type", "=", "change").where("direction", "=", "revert").where("status", "=", "success").where("config_name", "=", this.#configName).orderBy("id", "desc").limit(1).executeTakeFirst()
|
|
2990
3223
|
);
|
|
2991
3224
|
return {
|
|
2992
3225
|
name: record.name,
|
|
@@ -3007,7 +3240,7 @@ var ChangeHistory = class {
|
|
|
3007
3240
|
async getAllStatuses() {
|
|
3008
3241
|
const statuses = /* @__PURE__ */ new Map();
|
|
3009
3242
|
const [records, err] = await attempt(
|
|
3010
|
-
() => this.#
|
|
3243
|
+
() => this.#ndb.selectFrom(this.#tables.change).select(["id", "name", "status", "executed_at", "executed_by", "error_message"]).where("change_type", "=", "change").where("direction", "=", "change").where("config_name", "=", this.#configName).orderBy("id", "desc").execute()
|
|
3011
3244
|
);
|
|
3012
3245
|
if (err) {
|
|
3013
3246
|
observer.emit("error", {
|
|
@@ -3031,7 +3264,7 @@ var ChangeHistory = class {
|
|
|
3031
3264
|
}
|
|
3032
3265
|
}
|
|
3033
3266
|
const [reverts] = await attempt(
|
|
3034
|
-
() => this.#
|
|
3267
|
+
() => this.#ndb.selectFrom(this.#tables.change).select(["name", "executed_at"]).where("change_type", "=", "change").where("direction", "=", "revert").where("status", "=", "success").where("config_name", "=", this.#configName).orderBy("id", "desc").execute()
|
|
3035
3268
|
);
|
|
3036
3269
|
if (reverts) {
|
|
3037
3270
|
const seenReverts = /* @__PURE__ */ new Set();
|
|
@@ -3061,7 +3294,7 @@ var ChangeHistory = class {
|
|
|
3061
3294
|
return { needsRun: true, reason: "force" };
|
|
3062
3295
|
}
|
|
3063
3296
|
const [record, err] = await attempt(
|
|
3064
|
-
() => this.#
|
|
3297
|
+
() => this.#ndb.selectFrom(this.#tables.change).select(["status", "checksum"]).where("name", "=", name).where("change_type", "=", "change").where("direction", "=", "change").where("config_name", "=", this.#configName).orderBy("id", "desc").limit(1).executeTakeFirst()
|
|
3065
3298
|
);
|
|
3066
3299
|
if (err) {
|
|
3067
3300
|
observer.emit("error", {
|
|
@@ -3126,7 +3359,7 @@ var ChangeHistory = class {
|
|
|
3126
3359
|
*/
|
|
3127
3360
|
async createOperation(data2) {
|
|
3128
3361
|
const [result, err] = await attempt(
|
|
3129
|
-
() => this.#
|
|
3362
|
+
() => this.#ndb.insertInto(this.#tables.change).values({
|
|
3130
3363
|
name: data2.name,
|
|
3131
3364
|
change_type: "change",
|
|
3132
3365
|
direction: data2.direction,
|
|
@@ -3170,7 +3403,7 @@ var ChangeHistory = class {
|
|
|
3170
3403
|
status: "pending"
|
|
3171
3404
|
}));
|
|
3172
3405
|
const [, err] = await attempt(
|
|
3173
|
-
() => this.#
|
|
3406
|
+
() => this.#ndb.insertInto(this.#tables.executions).values(values).execute()
|
|
3174
3407
|
);
|
|
3175
3408
|
if (err) {
|
|
3176
3409
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -3193,7 +3426,7 @@ var ChangeHistory = class {
|
|
|
3193
3426
|
*/
|
|
3194
3427
|
async updateFileExecution(operationId, filepath, status, durationMs, errorMessage, skipReason) {
|
|
3195
3428
|
const [result, err] = await attempt(
|
|
3196
|
-
() => this.#
|
|
3429
|
+
() => this.#ndb.updateTable(this.#tables.executions).set({
|
|
3197
3430
|
status,
|
|
3198
3431
|
duration_ms: Math.round(durationMs),
|
|
3199
3432
|
error_message: errorMessage ?? "",
|
|
@@ -3228,7 +3461,7 @@ var ChangeHistory = class {
|
|
|
3228
3461
|
*/
|
|
3229
3462
|
async skipRemainingFiles(operationId, reason) {
|
|
3230
3463
|
const [, err] = await attempt(
|
|
3231
|
-
() => this.#
|
|
3464
|
+
() => this.#ndb.updateTable(this.#tables.executions).set({
|
|
3232
3465
|
status: "skipped",
|
|
3233
3466
|
skip_reason: reason
|
|
3234
3467
|
}).where("change_id", "=", operationId).where("status", "=", "pending").execute()
|
|
@@ -3251,7 +3484,7 @@ var ChangeHistory = class {
|
|
|
3251
3484
|
async finalizeOperation(operationId, status, checksum, durationMs, errorMessage) {
|
|
3252
3485
|
const truncatedError = errorMessage ? errorMessage.slice(0, 2e3) : "";
|
|
3253
3486
|
const [result, err] = await attempt(
|
|
3254
|
-
() => this.#
|
|
3487
|
+
() => this.#ndb.updateTable(this.#tables.change).set({
|
|
3255
3488
|
status,
|
|
3256
3489
|
checksum,
|
|
3257
3490
|
duration_ms: Math.round(durationMs),
|
|
@@ -3292,7 +3525,7 @@ var ChangeHistory = class {
|
|
|
3292
3525
|
*/
|
|
3293
3526
|
async recordReset(executedBy, reason) {
|
|
3294
3527
|
const [result, err] = await attempt(
|
|
3295
|
-
() => this.#
|
|
3528
|
+
() => this.#ndb.insertInto(this.#tables.change).values({
|
|
3296
3529
|
name: "__reset__",
|
|
3297
3530
|
change_type: "change",
|
|
3298
3531
|
direction: "change",
|
|
@@ -3322,14 +3555,14 @@ var ChangeHistory = class {
|
|
|
3322
3555
|
*/
|
|
3323
3556
|
async deleteRecords(name) {
|
|
3324
3557
|
const [operations, queryErr] = await attempt(
|
|
3325
|
-
() => this.#
|
|
3558
|
+
() => this.#ndb.selectFrom(this.#tables.change).select(["id"]).where("name", "=", name).where("change_type", "=", "change").where("config_name", "=", this.#configName).execute()
|
|
3326
3559
|
);
|
|
3327
3560
|
if (queryErr || !operations || operations.length === 0) {
|
|
3328
3561
|
return;
|
|
3329
3562
|
}
|
|
3330
3563
|
const operationIds = operations.map((o) => o.id);
|
|
3331
3564
|
const [, execErr] = await attempt(
|
|
3332
|
-
() => this.#
|
|
3565
|
+
() => this.#ndb.deleteFrom(this.#tables.executions).where("change_id", "in", operationIds).execute()
|
|
3333
3566
|
);
|
|
3334
3567
|
if (execErr) {
|
|
3335
3568
|
observer.emit("error", {
|
|
@@ -3339,7 +3572,7 @@ var ChangeHistory = class {
|
|
|
3339
3572
|
});
|
|
3340
3573
|
}
|
|
3341
3574
|
const [, changeErr] = await attempt(
|
|
3342
|
-
() => this.#
|
|
3575
|
+
() => this.#ndb.deleteFrom(this.#tables.change).where("id", "in", operationIds).execute()
|
|
3343
3576
|
);
|
|
3344
3577
|
if (changeErr) {
|
|
3345
3578
|
observer.emit("error", {
|
|
@@ -3359,7 +3592,7 @@ var ChangeHistory = class {
|
|
|
3359
3592
|
* @param limit - Max records to return
|
|
3360
3593
|
*/
|
|
3361
3594
|
async getHistory(name, limit) {
|
|
3362
|
-
let query = this.#
|
|
3595
|
+
let query = this.#ndb.selectFrom(this.#tables.change).select([
|
|
3363
3596
|
"id",
|
|
3364
3597
|
"name",
|
|
3365
3598
|
"direction",
|
|
@@ -3407,7 +3640,7 @@ var ChangeHistory = class {
|
|
|
3407
3640
|
* @param limit - Max records to return
|
|
3408
3641
|
*/
|
|
3409
3642
|
async getUnifiedHistory(changeTypes, limit) {
|
|
3410
|
-
let query = this.#
|
|
3643
|
+
let query = this.#ndb.selectFrom(this.#tables.change).select([
|
|
3411
3644
|
"id",
|
|
3412
3645
|
"name",
|
|
3413
3646
|
"change_type",
|
|
@@ -3463,7 +3696,7 @@ var ChangeHistory = class {
|
|
|
3463
3696
|
*/
|
|
3464
3697
|
async getFileHistory(operationId) {
|
|
3465
3698
|
const [records, err] = await attempt(
|
|
3466
|
-
() => this.#
|
|
3699
|
+
() => this.#ndb.selectFrom(this.#tables.executions).select([
|
|
3467
3700
|
"id",
|
|
3468
3701
|
"change_id",
|
|
3469
3702
|
"filepath",
|
|
@@ -3513,9 +3746,13 @@ var ChangeHistory = class {
|
|
|
3513
3746
|
};
|
|
3514
3747
|
var Tracker = class {
|
|
3515
3748
|
#db;
|
|
3749
|
+
#ndb;
|
|
3750
|
+
#tables;
|
|
3516
3751
|
#configName;
|
|
3517
|
-
constructor(db, configName) {
|
|
3752
|
+
constructor(db, configName, dialect = "sqlite") {
|
|
3518
3753
|
this.#db = db;
|
|
3754
|
+
this.#ndb = noormDb(db, dialect);
|
|
3755
|
+
this.#tables = getNoormTables(dialect);
|
|
3519
3756
|
this.#configName = configName;
|
|
3520
3757
|
}
|
|
3521
3758
|
/**
|
|
@@ -3538,15 +3775,16 @@ var Tracker = class {
|
|
|
3538
3775
|
return { needsRun: true, reason: "force" };
|
|
3539
3776
|
}
|
|
3540
3777
|
const [record, err] = await attempt(
|
|
3541
|
-
() => this.#
|
|
3542
|
-
|
|
3543
|
-
`${
|
|
3544
|
-
`${
|
|
3778
|
+
() => this.#ndb.selectFrom(this.#tables.executions).innerJoin(
|
|
3779
|
+
this.#tables.change,
|
|
3780
|
+
`${this.#tables.change}.id`,
|
|
3781
|
+
`${this.#tables.executions}.change_id`
|
|
3782
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
3545
3783
|
).select((eb) => [
|
|
3546
|
-
eb.ref(`${
|
|
3547
|
-
eb.ref(`${
|
|
3548
|
-
eb.ref(`${
|
|
3549
|
-
]).where(`${
|
|
3784
|
+
eb.ref(`${this.#tables.executions}.checksum`).as("checksum"),
|
|
3785
|
+
eb.ref(`${this.#tables.executions}.status`).as("exec_status"),
|
|
3786
|
+
eb.ref(`${this.#tables.change}.status`).as("change_status")
|
|
3787
|
+
]).where(`${this.#tables.executions}.filepath`, "=", filepath).where(`${this.#tables.change}.config_name`, "=", this.#configName).orderBy(`${this.#tables.executions}.id`, "desc").limit(1).executeTakeFirst()
|
|
3550
3788
|
);
|
|
3551
3789
|
if (err) {
|
|
3552
3790
|
observer.emit("error", {
|
|
@@ -3602,7 +3840,7 @@ var Tracker = class {
|
|
|
3602
3840
|
const direction = data2.direction ?? "commit";
|
|
3603
3841
|
const dbDirection = direction === "commit" ? "change" : "revert";
|
|
3604
3842
|
const [result, err] = await attempt(
|
|
3605
|
-
() => this.#
|
|
3843
|
+
() => this.#ndb.insertInto(this.#tables.change).values({
|
|
3606
3844
|
name: data2.name,
|
|
3607
3845
|
change_type: data2.changeType,
|
|
3608
3846
|
direction: dbDirection,
|
|
@@ -3639,7 +3877,7 @@ var Tracker = class {
|
|
|
3639
3877
|
*/
|
|
3640
3878
|
async recordExecution(data2) {
|
|
3641
3879
|
const [, err] = await attempt(
|
|
3642
|
-
() => this.#
|
|
3880
|
+
() => this.#ndb.insertInto(this.#tables.executions).values({
|
|
3643
3881
|
change_id: data2.changeId,
|
|
3644
3882
|
filepath: data2.filepath,
|
|
3645
3883
|
file_type: "sql",
|
|
@@ -3673,7 +3911,7 @@ var Tracker = class {
|
|
|
3673
3911
|
async finalizeOperation(operationId, status, durationMs, checksum, errorMessage) {
|
|
3674
3912
|
const truncatedError = errorMessage ? errorMessage.slice(0, 2e3) : "";
|
|
3675
3913
|
const [result, err] = await attempt(
|
|
3676
|
-
() => this.#
|
|
3914
|
+
() => this.#ndb.updateTable(this.#tables.change).set({
|
|
3677
3915
|
status,
|
|
3678
3916
|
duration_ms: Math.round(durationMs),
|
|
3679
3917
|
checksum: checksum ?? "",
|
|
@@ -3724,7 +3962,7 @@ var Tracker = class {
|
|
|
3724
3962
|
status: "pending"
|
|
3725
3963
|
}));
|
|
3726
3964
|
const [, err] = await attempt(
|
|
3727
|
-
() => this.#
|
|
3965
|
+
() => this.#ndb.insertInto(this.#tables.executions).values(values).execute()
|
|
3728
3966
|
);
|
|
3729
3967
|
if (err) {
|
|
3730
3968
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -3752,7 +3990,7 @@ var Tracker = class {
|
|
|
3752
3990
|
*/
|
|
3753
3991
|
async updateFileExecution(operationId, filepath, status, durationMs, errorMessage, skipReason) {
|
|
3754
3992
|
const [result, err] = await attempt(
|
|
3755
|
-
() => this.#
|
|
3993
|
+
() => this.#ndb.updateTable(this.#tables.executions).set({
|
|
3756
3994
|
status,
|
|
3757
3995
|
duration_ms: Math.round(durationMs),
|
|
3758
3996
|
error_message: errorMessage ?? "",
|
|
@@ -3792,7 +4030,7 @@ var Tracker = class {
|
|
|
3792
4030
|
*/
|
|
3793
4031
|
async skipRemainingFiles(operationId, reason) {
|
|
3794
4032
|
const [, err] = await attempt(
|
|
3795
|
-
() => this.#
|
|
4033
|
+
() => this.#ndb.updateTable(this.#tables.executions).set({
|
|
3796
4034
|
status: "skipped",
|
|
3797
4035
|
skip_reason: reason
|
|
3798
4036
|
}).where("change_id", "=", operationId).where("status", "=", "pending").execute()
|
|
@@ -3823,7 +4061,7 @@ var Tracker = class {
|
|
|
3823
4061
|
return { needsRun: true, reason: "force" };
|
|
3824
4062
|
}
|
|
3825
4063
|
const [record, err] = await attempt(
|
|
3826
|
-
() => this.#
|
|
4064
|
+
() => this.#ndb.selectFrom(this.#tables.change).select(["status", "checksum"]).where("name", "=", name).where("direction", "=", "change").where("config_name", "=", this.#configName).orderBy("id", "desc").limit(1).executeTakeFirst()
|
|
3827
4065
|
);
|
|
3828
4066
|
if (err) {
|
|
3829
4067
|
observer.emit("error", {
|
|
@@ -3877,11 +4115,13 @@ var Tracker = class {
|
|
|
3877
4115
|
var ChangeTracker = class extends Tracker {
|
|
3878
4116
|
// Store db and configName for change-specific queries
|
|
3879
4117
|
// (Tracker uses private fields, so we need our own references)
|
|
3880
|
-
#
|
|
4118
|
+
#ndb;
|
|
4119
|
+
#tables;
|
|
3881
4120
|
#configName;
|
|
3882
|
-
constructor(db, configName) {
|
|
3883
|
-
super(db, configName);
|
|
3884
|
-
this.#
|
|
4121
|
+
constructor(db, configName, dialect = "sqlite") {
|
|
4122
|
+
super(db, configName, dialect);
|
|
4123
|
+
this.#ndb = noormDb(db, dialect);
|
|
4124
|
+
this.#tables = getNoormTables(dialect);
|
|
3885
4125
|
this.#configName = configName;
|
|
3886
4126
|
}
|
|
3887
4127
|
/**
|
|
@@ -3893,7 +4133,7 @@ var ChangeTracker = class extends Tracker {
|
|
|
3893
4133
|
*/
|
|
3894
4134
|
async canRevert(name, force) {
|
|
3895
4135
|
const [record, err] = await attempt(
|
|
3896
|
-
() => this.#
|
|
4136
|
+
() => this.#ndb.selectFrom(this.#tables.change).select(["status"]).where("name", "=", name).where("change_type", "=", "change").where("direction", "=", "change").where("config_name", "=", this.#configName).orderBy("id", "desc").limit(1).executeTakeFirst()
|
|
3897
4137
|
);
|
|
3898
4138
|
if (err) {
|
|
3899
4139
|
observer.emit("error", {
|
|
@@ -3933,11 +4173,11 @@ var ChangeTracker = class extends Tracker {
|
|
|
3933
4173
|
*/
|
|
3934
4174
|
async markAsReverted(name) {
|
|
3935
4175
|
const [record] = await attempt(
|
|
3936
|
-
() => this.#
|
|
4176
|
+
() => this.#ndb.selectFrom(this.#tables.change).select(["id"]).where("name", "=", name).where("change_type", "=", "change").where("direction", "=", "change").where("config_name", "=", this.#configName).orderBy("id", "desc").limit(1).executeTakeFirst()
|
|
3937
4177
|
);
|
|
3938
4178
|
if (record) {
|
|
3939
4179
|
await attempt(
|
|
3940
|
-
() => this.#
|
|
4180
|
+
() => this.#ndb.updateTable(this.#tables.change).set({ status: "reverted" }).where("id", "=", record.id).execute()
|
|
3941
4181
|
);
|
|
3942
4182
|
}
|
|
3943
4183
|
}
|
|
@@ -3953,7 +4193,7 @@ var ChangeTracker = class extends Tracker {
|
|
|
3953
4193
|
*/
|
|
3954
4194
|
async markAllAsStale() {
|
|
3955
4195
|
const [result, err] = await attempt(
|
|
3956
|
-
() => this.#
|
|
4196
|
+
() => this.#ndb.updateTable(this.#tables.change).set({ status: "stale" }).where("direction", "=", "change").where("status", "in", ["success", "failed", "pending"]).where("config_name", "=", this.#configName).execute()
|
|
3957
4197
|
);
|
|
3958
4198
|
if (err) {
|
|
3959
4199
|
observer.emit("error", {
|
|
@@ -4065,7 +4305,7 @@ var LockManager = class {
|
|
|
4065
4305
|
observer.emit("lock:acquiring", { configName, identity });
|
|
4066
4306
|
while (true) {
|
|
4067
4307
|
await this.cleanupExpired(db, configName, opts.dialect);
|
|
4068
|
-
const existing = await this.getLock(db, configName);
|
|
4308
|
+
const existing = await this.getLock(db, configName, opts.dialect);
|
|
4069
4309
|
if (!existing) {
|
|
4070
4310
|
const lock = await this.createLock(db, configName, identity, opts);
|
|
4071
4311
|
observer.emit("lock:acquired", {
|
|
@@ -4117,18 +4357,21 @@ var LockManager = class {
|
|
|
4117
4357
|
* @param db - Kysely database instance
|
|
4118
4358
|
* @param configName - Config/database scope
|
|
4119
4359
|
* @param identity - Identity of the lock holder
|
|
4360
|
+
* @param dialect - Database dialect for schema-aware table resolution
|
|
4120
4361
|
* @throws LockNotFoundError if no lock exists
|
|
4121
4362
|
* @throws LockOwnershipError if lock is held by someone else
|
|
4122
4363
|
*/
|
|
4123
|
-
async release(db, configName, identity) {
|
|
4124
|
-
const
|
|
4364
|
+
async release(db, configName, identity, dialect) {
|
|
4365
|
+
const ndb = noormDb(db, dialect);
|
|
4366
|
+
const tables = getNoormTables(dialect);
|
|
4367
|
+
const existing = await this.getLock(db, configName, dialect);
|
|
4125
4368
|
if (!existing) {
|
|
4126
4369
|
throw new LockNotFoundError(configName, identity);
|
|
4127
4370
|
}
|
|
4128
4371
|
if (existing.lockedBy !== identity) {
|
|
4129
4372
|
throw new LockOwnershipError(configName, identity, existing.lockedBy);
|
|
4130
4373
|
}
|
|
4131
|
-
await
|
|
4374
|
+
await ndb.deleteFrom(tables.lock).where("config_name", "=", configName).where("locked_by", "=", identity).execute();
|
|
4132
4375
|
observer.emit("lock:released", { configName, identity });
|
|
4133
4376
|
}
|
|
4134
4377
|
/**
|
|
@@ -4139,14 +4382,17 @@ var LockManager = class {
|
|
|
4139
4382
|
*
|
|
4140
4383
|
* @param db - Kysely database instance
|
|
4141
4384
|
* @param configName - Config/database scope
|
|
4385
|
+
* @param dialect - Database dialect for schema-aware table resolution
|
|
4142
4386
|
* @returns true if a lock was released, false if none existed
|
|
4143
4387
|
*/
|
|
4144
|
-
async forceRelease(db, configName) {
|
|
4145
|
-
const
|
|
4388
|
+
async forceRelease(db, configName, dialect) {
|
|
4389
|
+
const ndb = noormDb(db, dialect);
|
|
4390
|
+
const tables = getNoormTables(dialect);
|
|
4391
|
+
const existing = await this.getLock(db, configName, dialect);
|
|
4146
4392
|
if (!existing) {
|
|
4147
4393
|
return false;
|
|
4148
4394
|
}
|
|
4149
|
-
await
|
|
4395
|
+
await ndb.deleteFrom(tables.lock).where("config_name", "=", configName).execute();
|
|
4150
4396
|
observer.emit("lock:released", {
|
|
4151
4397
|
configName,
|
|
4152
4398
|
identity: existing.lockedBy
|
|
@@ -4167,11 +4413,12 @@ var LockManager = class {
|
|
|
4167
4413
|
* @returns Result of the operation
|
|
4168
4414
|
*/
|
|
4169
4415
|
async withLock(db, configName, identity, operation, options = {}) {
|
|
4416
|
+
const opts = { ...DEFAULT_LOCK_OPTIONS, ...options };
|
|
4170
4417
|
await this.acquire(db, configName, identity, options);
|
|
4171
4418
|
try {
|
|
4172
4419
|
return await operation();
|
|
4173
4420
|
} finally {
|
|
4174
|
-
const [, err] = await attempt(() => this.release(db, configName, identity));
|
|
4421
|
+
const [, err] = await attempt(() => this.release(db, configName, identity, opts.dialect));
|
|
4175
4422
|
if (err) {
|
|
4176
4423
|
observer.emit("error", {
|
|
4177
4424
|
source: "lock",
|
|
@@ -4189,12 +4436,13 @@ var LockManager = class {
|
|
|
4189
4436
|
* @param db - Kysely database instance
|
|
4190
4437
|
* @param configName - Config/database scope
|
|
4191
4438
|
* @param identity - Expected lock holder
|
|
4439
|
+
* @param dialect - Database dialect for schema-aware table resolution
|
|
4192
4440
|
* @throws LockExpiredError if lock has expired
|
|
4193
4441
|
* @throws LockOwnershipError if lock is held by someone else
|
|
4194
4442
|
* @throws LockNotFoundError if no lock exists
|
|
4195
4443
|
*/
|
|
4196
4444
|
async validate(db, configName, identity, dialect = "postgres") {
|
|
4197
|
-
const existing = await this.getLock(db, configName);
|
|
4445
|
+
const existing = await this.getLock(db, configName, dialect);
|
|
4198
4446
|
if (!existing) {
|
|
4199
4447
|
throw new LockNotFoundError(configName, identity);
|
|
4200
4448
|
}
|
|
@@ -4216,7 +4464,8 @@ var LockManager = class {
|
|
|
4216
4464
|
* @returns Updated lock state
|
|
4217
4465
|
*/
|
|
4218
4466
|
async extend(db, configName, identity, options = {}) {
|
|
4219
|
-
|
|
4467
|
+
const opts = { ...DEFAULT_LOCK_OPTIONS, ...options };
|
|
4468
|
+
await this.validate(db, configName, identity, opts.dialect);
|
|
4220
4469
|
return this.extendLock(db, configName, identity, options);
|
|
4221
4470
|
}
|
|
4222
4471
|
/**
|
|
@@ -4229,7 +4478,7 @@ var LockManager = class {
|
|
|
4229
4478
|
*/
|
|
4230
4479
|
async status(db, configName, dialect = "postgres") {
|
|
4231
4480
|
await this.cleanupExpired(db, configName, dialect);
|
|
4232
|
-
const lock = await this.getLock(db, configName);
|
|
4481
|
+
const lock = await this.getLock(db, configName, dialect);
|
|
4233
4482
|
return {
|
|
4234
4483
|
isLocked: lock !== null,
|
|
4235
4484
|
lock
|
|
@@ -4241,8 +4490,10 @@ var LockManager = class {
|
|
|
4241
4490
|
/**
|
|
4242
4491
|
* Get lock from database, or null if none exists.
|
|
4243
4492
|
*/
|
|
4244
|
-
async getLock(db, configName) {
|
|
4245
|
-
const
|
|
4493
|
+
async getLock(db, configName, dialect) {
|
|
4494
|
+
const ndb = noormDb(db, dialect);
|
|
4495
|
+
const tables = getNoormTables(dialect);
|
|
4496
|
+
const row = await ndb.selectFrom(tables.lock).selectAll().where("config_name", "=", configName).executeTakeFirst();
|
|
4246
4497
|
if (!row) {
|
|
4247
4498
|
return null;
|
|
4248
4499
|
}
|
|
@@ -4257,9 +4508,11 @@ var LockManager = class {
|
|
|
4257
4508
|
* Create a new lock in the database.
|
|
4258
4509
|
*/
|
|
4259
4510
|
async createLock(db, configName, identity, opts) {
|
|
4511
|
+
const ndb = noormDb(db, opts.dialect);
|
|
4512
|
+
const tables = getNoormTables(opts.dialect);
|
|
4260
4513
|
const now = /* @__PURE__ */ new Date();
|
|
4261
4514
|
const expiresAt = new Date(now.getTime() + opts.timeout);
|
|
4262
|
-
await
|
|
4515
|
+
await ndb.insertInto(tables.lock).values({
|
|
4263
4516
|
config_name: configName,
|
|
4264
4517
|
locked_by: identity,
|
|
4265
4518
|
locked_at: formatDateForDialect(now, opts.dialect),
|
|
@@ -4279,6 +4532,8 @@ var LockManager = class {
|
|
|
4279
4532
|
async extendLock(db, configName, identity, opts) {
|
|
4280
4533
|
const dialect = opts.dialect ?? DEFAULT_LOCK_OPTIONS.dialect;
|
|
4281
4534
|
const timeout = opts.timeout ?? DEFAULT_LOCK_OPTIONS.timeout;
|
|
4535
|
+
const ndb = noormDb(db, dialect);
|
|
4536
|
+
const tables = getNoormTables(dialect);
|
|
4282
4537
|
const now = /* @__PURE__ */ new Date();
|
|
4283
4538
|
const expiresAt = new Date(now.getTime() + timeout);
|
|
4284
4539
|
const updateValues = {
|
|
@@ -4287,19 +4542,21 @@ var LockManager = class {
|
|
|
4287
4542
|
if (opts.reason !== void 0) {
|
|
4288
4543
|
updateValues["reason"] = opts.reason;
|
|
4289
4544
|
}
|
|
4290
|
-
await
|
|
4291
|
-
const lock = await this.getLock(db, configName);
|
|
4545
|
+
await ndb.updateTable(tables.lock).set(updateValues).where("config_name", "=", configName).where("locked_by", "=", identity).execute();
|
|
4546
|
+
const lock = await this.getLock(db, configName, dialect);
|
|
4292
4547
|
return lock;
|
|
4293
4548
|
}
|
|
4294
4549
|
/**
|
|
4295
4550
|
* Clean up expired locks.
|
|
4296
4551
|
*/
|
|
4297
4552
|
async cleanupExpired(db, configName, dialect = "postgres") {
|
|
4553
|
+
const ndb = noormDb(db, dialect);
|
|
4554
|
+
const tables = getNoormTables(dialect);
|
|
4298
4555
|
const now = /* @__PURE__ */ new Date();
|
|
4299
4556
|
const nowForDb = formatDateForDialect(now, dialect);
|
|
4300
|
-
const expired = await
|
|
4557
|
+
const expired = await ndb.selectFrom(tables.lock).select(["locked_by"]).where("config_name", "=", configName).where("expires_at", "<", nowForDb).executeTakeFirst();
|
|
4301
4558
|
if (expired) {
|
|
4302
|
-
await
|
|
4559
|
+
await ndb.deleteFrom(tables.lock).where("config_name", "=", configName).where("expires_at", "<", nowForDb).execute();
|
|
4303
4560
|
observer.emit("lock:expired", {
|
|
4304
4561
|
configName,
|
|
4305
4562
|
previousHolder: expired.locked_by
|
|
@@ -4315,7 +4572,7 @@ function getLockManager() {
|
|
|
4315
4572
|
return instance3;
|
|
4316
4573
|
}
|
|
4317
4574
|
function sleep(ms) {
|
|
4318
|
-
return new Promise((
|
|
4575
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
4319
4576
|
}
|
|
4320
4577
|
|
|
4321
4578
|
// src/core/change/executor.ts
|
|
@@ -4358,7 +4615,7 @@ async function executeChange(context, change, options = {}) {
|
|
|
4358
4615
|
return createFailedResult(change.name, "change", checksumErr.message, start);
|
|
4359
4616
|
}
|
|
4360
4617
|
const changeChecksum = computeCombinedChecksum(checksums);
|
|
4361
|
-
const history = new ChangeHistory(context.db, context.configName);
|
|
4618
|
+
const history = new ChangeHistory(context.db, context.configName, context.dialect ?? "postgres");
|
|
4362
4619
|
if (!opts.dryRun && !opts.preview) {
|
|
4363
4620
|
const needsRunResult = await history.needsRun(
|
|
4364
4621
|
change.name,
|
|
@@ -4407,7 +4664,7 @@ async function executeChange(context, change, options = {}) {
|
|
|
4407
4664
|
start
|
|
4408
4665
|
)
|
|
4409
4666
|
);
|
|
4410
|
-
await attempt(() => lockManager.release(context.db, context.configName, identity));
|
|
4667
|
+
await attempt(() => lockManager.release(context.db, context.configName, identity, context.dialect ?? "postgres"));
|
|
4411
4668
|
if (execErr) {
|
|
4412
4669
|
throw execErr;
|
|
4413
4670
|
}
|
|
@@ -4430,8 +4687,8 @@ async function revertChange(context, change, options = {}) {
|
|
|
4430
4687
|
return createFailedResult(change.name, "revert", checksumErr.message, start);
|
|
4431
4688
|
}
|
|
4432
4689
|
const revertChecksum = computeCombinedChecksum(checksums);
|
|
4433
|
-
const history = new ChangeHistory(context.db, context.configName);
|
|
4434
|
-
const tracker = new ChangeTracker(context.db, context.configName);
|
|
4690
|
+
const history = new ChangeHistory(context.db, context.configName, context.dialect ?? "postgres");
|
|
4691
|
+
const tracker = new ChangeTracker(context.db, context.configName, context.dialect ?? "postgres");
|
|
4435
4692
|
if (!opts.dryRun && !opts.preview) {
|
|
4436
4693
|
const canRevertResult = await tracker.canRevert(change.name, opts.force);
|
|
4437
4694
|
if (!canRevertResult.canRevert) {
|
|
@@ -4479,7 +4736,7 @@ async function revertChange(context, change, options = {}) {
|
|
|
4479
4736
|
start
|
|
4480
4737
|
)
|
|
4481
4738
|
);
|
|
4482
|
-
await attempt(() => lockManager.release(context.db, context.configName, identity));
|
|
4739
|
+
await attempt(() => lockManager.release(context.db, context.configName, identity, context.dialect ?? "postgres"));
|
|
4483
4740
|
if (execErr) {
|
|
4484
4741
|
throw execErr;
|
|
4485
4742
|
}
|
|
@@ -4893,7 +5150,7 @@ var ChangeManager = class {
|
|
|
4893
5150
|
#history;
|
|
4894
5151
|
constructor(context) {
|
|
4895
5152
|
this.#context = context;
|
|
4896
|
-
this.#history = new ChangeHistory(context.db, context.configName);
|
|
5153
|
+
this.#history = new ChangeHistory(context.db, context.configName, context.dialect ?? "postgres");
|
|
4897
5154
|
}
|
|
4898
5155
|
// ─────────────────────────────────────────────────────────
|
|
4899
5156
|
// List
|
|
@@ -5513,7 +5770,7 @@ async function runFile(context, filepath, options = {}) {
|
|
|
5513
5770
|
filepath,
|
|
5514
5771
|
configName: context.configName
|
|
5515
5772
|
});
|
|
5516
|
-
const tracker = new Tracker(context.db, context.configName);
|
|
5773
|
+
const tracker = new Tracker(context.db, context.configName, context.dialect ?? "postgres");
|
|
5517
5774
|
const operationName = `run:${(/* @__PURE__ */ new Date()).toISOString()}`;
|
|
5518
5775
|
const [operationId, createErr] = await attempt(
|
|
5519
5776
|
() => tracker.createOperation({
|
|
@@ -5645,7 +5902,7 @@ async function executeFiles2(context, files, runOptions, execOptions) {
|
|
|
5645
5902
|
durationMs: durationMs2
|
|
5646
5903
|
};
|
|
5647
5904
|
}
|
|
5648
|
-
const tracker = execOptions.tracker ?? new Tracker(context.db, context.configName);
|
|
5905
|
+
const tracker = execOptions.tracker ?? new Tracker(context.db, context.configName, context.dialect ?? "postgres");
|
|
5649
5906
|
const [operationId, createErr] = await attempt(
|
|
5650
5907
|
() => tracker.createOperation({
|
|
5651
5908
|
name: execOptions.operationName,
|
|
@@ -6231,13 +6488,14 @@ var RunNamespace = class {
|
|
|
6231
6488
|
configName: this.#state.config.name,
|
|
6232
6489
|
identity: this.#state.identity,
|
|
6233
6490
|
projectRoot: this.#state.projectRoot,
|
|
6491
|
+
dialect: this.#state.config.connection.dialect,
|
|
6234
6492
|
config: this.#state.config,
|
|
6235
6493
|
secrets: state.getAllSecrets(this.#state.config.name),
|
|
6236
6494
|
globalSecrets: state.getAllGlobalSecrets()
|
|
6237
6495
|
};
|
|
6238
6496
|
}
|
|
6239
6497
|
};
|
|
6240
|
-
var EXCLUDED_SCHEMAS = ["pg_catalog", "information_schema", "pg_toast"];
|
|
6498
|
+
var EXCLUDED_SCHEMAS = ["pg_catalog", "information_schema", "pg_toast", "noorm"];
|
|
6241
6499
|
var postgresExploreOperations = {
|
|
6242
6500
|
async getOverview(db) {
|
|
6243
6501
|
const [tables, views, procedures, functions, types, indexes, foreignKeys] = await Promise.all([
|
|
@@ -7435,7 +7693,7 @@ var mysqlExploreOperations = {
|
|
|
7435
7693
|
};
|
|
7436
7694
|
}
|
|
7437
7695
|
};
|
|
7438
|
-
var EXCLUDED_SCHEMAS2 = ["sys", "INFORMATION_SCHEMA", "guest"];
|
|
7696
|
+
var EXCLUDED_SCHEMAS2 = ["sys", "INFORMATION_SCHEMA", "guest", "noorm"];
|
|
7439
7697
|
var mssqlExploreOperations = {
|
|
7440
7698
|
async getOverview(db) {
|
|
7441
7699
|
const [tables, views, procedures, functions, types, indexes, foreignKeys] = await Promise.all([
|
|
@@ -8649,10 +8907,10 @@ function getTeardownOperations(dialect) {
|
|
|
8649
8907
|
}
|
|
8650
8908
|
|
|
8651
8909
|
// src/core/teardown/operations.ts
|
|
8652
|
-
var NOORM_TABLE_NAMES = new Set(Object.values(NOORM_TABLES));
|
|
8653
8910
|
function isNoormTable2(name) {
|
|
8654
8911
|
if (!name) return false;
|
|
8655
|
-
|
|
8912
|
+
if (name.startsWith("__noorm_")) return true;
|
|
8913
|
+
return false;
|
|
8656
8914
|
}
|
|
8657
8915
|
async function truncateData(db, dialect, options = {}) {
|
|
8658
8916
|
const startTime = performance.now();
|
|
@@ -8825,11 +9083,13 @@ async function teardownSchema(db, dialect, options = {}) {
|
|
|
8825
9083
|
if (options.configName && options.executedBy && !options.dryRun) {
|
|
8826
9084
|
const tracker = new ChangeTracker(
|
|
8827
9085
|
db,
|
|
8828
|
-
options.configName
|
|
9086
|
+
options.configName,
|
|
9087
|
+
dialect
|
|
8829
9088
|
);
|
|
8830
9089
|
const history = new ChangeHistory(
|
|
8831
9090
|
db,
|
|
8832
|
-
options.configName
|
|
9091
|
+
options.configName,
|
|
9092
|
+
dialect
|
|
8833
9093
|
);
|
|
8834
9094
|
result.staleCount = await tracker.markAllAsStale();
|
|
8835
9095
|
const parts = [
|
|
@@ -9058,7 +9318,8 @@ var LockNamespace = class {
|
|
|
9058
9318
|
await lockManager.release(
|
|
9059
9319
|
this.#kysely,
|
|
9060
9320
|
this.#state.config.name,
|
|
9061
|
-
identityStr
|
|
9321
|
+
identityStr,
|
|
9322
|
+
this.#state.config.connection.dialect
|
|
9062
9323
|
);
|
|
9063
9324
|
}
|
|
9064
9325
|
/**
|
|
@@ -9110,7 +9371,8 @@ var LockNamespace = class {
|
|
|
9110
9371
|
const lockManager = getLockManager();
|
|
9111
9372
|
return lockManager.forceRelease(
|
|
9112
9373
|
this.#kysely,
|
|
9113
|
-
this.#state.config.name
|
|
9374
|
+
this.#state.config.name,
|
|
9375
|
+
this.#state.config.connection.dialect
|
|
9114
9376
|
);
|
|
9115
9377
|
}
|
|
9116
9378
|
// ─────────────────────────────────────────────────────
|
|
@@ -9123,9 +9385,11 @@ var LockNamespace = class {
|
|
|
9123
9385
|
return this.#state.connection.db;
|
|
9124
9386
|
}
|
|
9125
9387
|
};
|
|
9126
|
-
async function getUsersWithoutVaultAccess(db) {
|
|
9388
|
+
async function getUsersWithoutVaultAccess(db, dialect) {
|
|
9389
|
+
const ndb = noormDb(db, dialect);
|
|
9390
|
+
const tables = getNoormTables(dialect);
|
|
9127
9391
|
const [rows, err] = await attempt(async () => {
|
|
9128
|
-
return
|
|
9392
|
+
return ndb.selectFrom(tables.identities).select([
|
|
9129
9393
|
"identity_hash",
|
|
9130
9394
|
"public_key",
|
|
9131
9395
|
"name",
|
|
@@ -9140,11 +9404,13 @@ async function getUsersWithoutVaultAccess(db) {
|
|
|
9140
9404
|
email: r.email
|
|
9141
9405
|
}));
|
|
9142
9406
|
}
|
|
9143
|
-
async function propagateVaultKey(db, vaultKey) {
|
|
9144
|
-
const
|
|
9407
|
+
async function propagateVaultKey(db, vaultKey, dialect) {
|
|
9408
|
+
const ndb = noormDb(db, dialect);
|
|
9409
|
+
const tables = getNoormTables(dialect);
|
|
9410
|
+
const users = await getUsersWithoutVaultAccess(db, dialect);
|
|
9145
9411
|
if (users.length === 0) {
|
|
9146
9412
|
const [countResult2] = await attempt(async () => {
|
|
9147
|
-
return
|
|
9413
|
+
return ndb.selectFrom(tables.identities).select(ndb.fn.count("id").as("count")).where("encrypted_vault_key", "is not", null).executeTakeFirst();
|
|
9148
9414
|
});
|
|
9149
9415
|
return {
|
|
9150
9416
|
propagatedTo: [],
|
|
@@ -9156,7 +9422,7 @@ async function propagateVaultKey(db, vaultKey) {
|
|
|
9156
9422
|
const encrypted = encryptVaultKey(vaultKey, user.publicKey);
|
|
9157
9423
|
const encryptedJson = JSON.stringify(encrypted);
|
|
9158
9424
|
const [, err] = await attempt(async () => {
|
|
9159
|
-
return
|
|
9425
|
+
return ndb.updateTable(tables.identities).set({ encrypted_vault_key: encryptedJson }).where("identity_hash", "=", user.identityHash).execute();
|
|
9160
9426
|
});
|
|
9161
9427
|
if (!err) {
|
|
9162
9428
|
propagatedTo.push(user.identityHash);
|
|
@@ -9167,7 +9433,7 @@ async function propagateVaultKey(db, vaultKey) {
|
|
|
9167
9433
|
}
|
|
9168
9434
|
}
|
|
9169
9435
|
const [countResult] = await attempt(async () => {
|
|
9170
|
-
return
|
|
9436
|
+
return ndb.selectFrom(tables.identities).select(ndb.fn.count("id").as("count")).where("encrypted_vault_key", "is not", null).executeTakeFirst();
|
|
9171
9437
|
});
|
|
9172
9438
|
const totalWithAccess = countResult ? Number(countResult.count) : 0;
|
|
9173
9439
|
const alreadyHadAccess = totalWithAccess - propagatedTo.length;
|
|
@@ -9247,18 +9513,20 @@ async function copyVaultSecrets(sourceConfig, destConfig, keys, identityHash, pr
|
|
|
9247
9513
|
const sourceVaultKey = await getVaultKey(
|
|
9248
9514
|
ctx.source.db,
|
|
9249
9515
|
identityHash,
|
|
9250
|
-
privateKey
|
|
9516
|
+
privateKey,
|
|
9517
|
+
ctx.source.dialect
|
|
9251
9518
|
);
|
|
9252
9519
|
if (!sourceVaultKey) {
|
|
9253
9520
|
throw new Error(`No vault access on source config "${sourceConfig.name}"`);
|
|
9254
9521
|
}
|
|
9255
|
-
const destStatus = await getVaultStatus(ctx.destination.db, identityHash);
|
|
9256
|
-
let destVaultKey
|
|
9522
|
+
const destStatus = await getVaultStatus(ctx.destination.db, identityHash, ctx.destination.dialect);
|
|
9523
|
+
let destVaultKey;
|
|
9257
9524
|
if (!destStatus.isInitialized) {
|
|
9258
9525
|
const [newKey, initErr] = await initializeVault(
|
|
9259
9526
|
ctx.destination.db,
|
|
9260
9527
|
identityHash,
|
|
9261
|
-
publicKey
|
|
9528
|
+
publicKey,
|
|
9529
|
+
ctx.destination.dialect
|
|
9262
9530
|
);
|
|
9263
9531
|
if (initErr) {
|
|
9264
9532
|
throw new Error(`Failed to initialize vault on destination: ${initErr.message}`);
|
|
@@ -9270,13 +9538,14 @@ async function copyVaultSecrets(sourceConfig, destConfig, keys, identityHash, pr
|
|
|
9270
9538
|
destVaultKey = await getVaultKey(
|
|
9271
9539
|
ctx.destination.db,
|
|
9272
9540
|
identityHash,
|
|
9273
|
-
privateKey
|
|
9541
|
+
privateKey,
|
|
9542
|
+
ctx.destination.dialect
|
|
9274
9543
|
);
|
|
9275
9544
|
}
|
|
9276
9545
|
if (!destVaultKey) {
|
|
9277
9546
|
throw new Error("Failed to get vault key for destination");
|
|
9278
9547
|
}
|
|
9279
|
-
const allSourceSecrets = await getAllVaultSecrets(ctx.source.db, sourceVaultKey);
|
|
9548
|
+
const allSourceSecrets = await getAllVaultSecrets(ctx.source.db, sourceVaultKey, ctx.source.dialect);
|
|
9280
9549
|
const secretsToCopy = keys === "all" ? Object.entries(allSourceSecrets) : Object.entries(allSourceSecrets).filter(([key]) => keys.includes(key));
|
|
9281
9550
|
const result = {
|
|
9282
9551
|
copied: [],
|
|
@@ -9294,7 +9563,7 @@ async function copyVaultSecrets(sourceConfig, destConfig, keys, identityHash, pr
|
|
|
9294
9563
|
}
|
|
9295
9564
|
}
|
|
9296
9565
|
for (const [key, secret] of secretsToCopy) {
|
|
9297
|
-
const exists = await vaultSecretExists(ctx.destination.db, key);
|
|
9566
|
+
const exists = await vaultSecretExists(ctx.destination.db, key, ctx.destination.dialect);
|
|
9298
9567
|
if (exists && !force) {
|
|
9299
9568
|
result.skipped.push(key);
|
|
9300
9569
|
continue;
|
|
@@ -9305,7 +9574,8 @@ async function copyVaultSecrets(sourceConfig, destConfig, keys, identityHash, pr
|
|
|
9305
9574
|
destVaultKey,
|
|
9306
9575
|
key,
|
|
9307
9576
|
secret.value,
|
|
9308
|
-
setBy
|
|
9577
|
+
setBy,
|
|
9578
|
+
ctx.destination.dialect
|
|
9309
9579
|
);
|
|
9310
9580
|
if (setErr) {
|
|
9311
9581
|
result.errors.push({ key, error: setErr.message });
|
|
@@ -9346,7 +9616,8 @@ var VaultNamespace = class {
|
|
|
9346
9616
|
return initializeVault(
|
|
9347
9617
|
this.#kysely,
|
|
9348
9618
|
this.#identityHash,
|
|
9349
|
-
this.#publicKey
|
|
9619
|
+
this.#publicKey,
|
|
9620
|
+
this.#dialect
|
|
9350
9621
|
);
|
|
9351
9622
|
}
|
|
9352
9623
|
/**
|
|
@@ -9360,7 +9631,8 @@ var VaultNamespace = class {
|
|
|
9360
9631
|
async status() {
|
|
9361
9632
|
return getVaultStatus(
|
|
9362
9633
|
this.#kysely,
|
|
9363
|
-
this.#identityHash
|
|
9634
|
+
this.#identityHash,
|
|
9635
|
+
this.#dialect
|
|
9364
9636
|
);
|
|
9365
9637
|
}
|
|
9366
9638
|
// ─────────────────────────────────────────────────────
|
|
@@ -9382,7 +9654,8 @@ var VaultNamespace = class {
|
|
|
9382
9654
|
vaultKey,
|
|
9383
9655
|
key,
|
|
9384
9656
|
value,
|
|
9385
|
-
formatIdentity(this.#state.identity)
|
|
9657
|
+
formatIdentity(this.#state.identity),
|
|
9658
|
+
this.#dialect
|
|
9386
9659
|
);
|
|
9387
9660
|
}
|
|
9388
9661
|
/**
|
|
@@ -9399,7 +9672,8 @@ var VaultNamespace = class {
|
|
|
9399
9672
|
return getVaultSecret(
|
|
9400
9673
|
this.#kysely,
|
|
9401
9674
|
vaultKey,
|
|
9402
|
-
key
|
|
9675
|
+
key,
|
|
9676
|
+
this.#dialect
|
|
9403
9677
|
);
|
|
9404
9678
|
}
|
|
9405
9679
|
/**
|
|
@@ -9415,7 +9689,8 @@ var VaultNamespace = class {
|
|
|
9415
9689
|
if (!vaultKey) return {};
|
|
9416
9690
|
return getAllVaultSecrets(
|
|
9417
9691
|
this.#kysely,
|
|
9418
|
-
vaultKey
|
|
9692
|
+
vaultKey,
|
|
9693
|
+
this.#dialect
|
|
9419
9694
|
);
|
|
9420
9695
|
}
|
|
9421
9696
|
/**
|
|
@@ -9428,7 +9703,8 @@ var VaultNamespace = class {
|
|
|
9428
9703
|
*/
|
|
9429
9704
|
async list() {
|
|
9430
9705
|
return listVaultSecretKeys(
|
|
9431
|
-
this.#kysely
|
|
9706
|
+
this.#kysely,
|
|
9707
|
+
this.#dialect
|
|
9432
9708
|
);
|
|
9433
9709
|
}
|
|
9434
9710
|
/**
|
|
@@ -9442,7 +9718,8 @@ var VaultNamespace = class {
|
|
|
9442
9718
|
async delete(key) {
|
|
9443
9719
|
return deleteVaultSecret(
|
|
9444
9720
|
this.#kysely,
|
|
9445
|
-
key
|
|
9721
|
+
key,
|
|
9722
|
+
this.#dialect
|
|
9446
9723
|
);
|
|
9447
9724
|
}
|
|
9448
9725
|
/**
|
|
@@ -9456,7 +9733,8 @@ var VaultNamespace = class {
|
|
|
9456
9733
|
async exists(key) {
|
|
9457
9734
|
return vaultSecretExists(
|
|
9458
9735
|
this.#kysely,
|
|
9459
|
-
key
|
|
9736
|
+
key,
|
|
9737
|
+
this.#dialect
|
|
9460
9738
|
);
|
|
9461
9739
|
}
|
|
9462
9740
|
// ─────────────────────────────────────────────────────
|
|
@@ -9477,7 +9755,8 @@ var VaultNamespace = class {
|
|
|
9477
9755
|
}
|
|
9478
9756
|
return propagateVaultKey(
|
|
9479
9757
|
this.#kysely,
|
|
9480
|
-
vaultKey
|
|
9758
|
+
vaultKey,
|
|
9759
|
+
this.#dialect
|
|
9481
9760
|
);
|
|
9482
9761
|
}
|
|
9483
9762
|
/**
|
|
@@ -9516,11 +9795,15 @@ var VaultNamespace = class {
|
|
|
9516
9795
|
const identity = this.#state.identity;
|
|
9517
9796
|
return identity["publicKey"] ?? "";
|
|
9518
9797
|
}
|
|
9798
|
+
get #dialect() {
|
|
9799
|
+
return this.#state.config.connection.dialect;
|
|
9800
|
+
}
|
|
9519
9801
|
async #getVaultKey(privateKey) {
|
|
9520
9802
|
return getVaultKey(
|
|
9521
9803
|
this.#kysely,
|
|
9522
9804
|
this.#identityHash,
|
|
9523
|
-
privateKey
|
|
9805
|
+
privateKey,
|
|
9806
|
+
this.#dialect
|
|
9524
9807
|
);
|
|
9525
9808
|
}
|
|
9526
9809
|
};
|
|
@@ -10424,16 +10707,16 @@ async function planTransfer(ctx, options = {}) {
|
|
|
10424
10707
|
const orderedNames = sortErr ? allTables.map((t) => t.name) : sortedNames;
|
|
10425
10708
|
const tablePlans = [];
|
|
10426
10709
|
for (const name of orderedNames) {
|
|
10427
|
-
const
|
|
10428
|
-
if (!
|
|
10710
|
+
const meta2 = tablesByName[name];
|
|
10711
|
+
if (!meta2) continue;
|
|
10429
10712
|
tablePlans.push({
|
|
10430
|
-
name:
|
|
10431
|
-
schema:
|
|
10432
|
-
rowCount:
|
|
10433
|
-
hasIdentity:
|
|
10434
|
-
identityColumn:
|
|
10435
|
-
primaryKey:
|
|
10436
|
-
columns:
|
|
10713
|
+
name: meta2.name,
|
|
10714
|
+
schema: meta2.schema,
|
|
10715
|
+
rowCount: meta2.rowCount,
|
|
10716
|
+
hasIdentity: meta2.hasIdentity,
|
|
10717
|
+
identityColumn: meta2.identityColumn,
|
|
10718
|
+
primaryKey: meta2.primaryKey,
|
|
10719
|
+
columns: meta2.columns,
|
|
10437
10720
|
dependsOn: dependencyMap.get(name) ?? []
|
|
10438
10721
|
});
|
|
10439
10722
|
}
|
|
@@ -10977,8 +11260,8 @@ async function executeTransfer(ctx, plan, options = {}) {
|
|
|
10977
11260
|
const strategy = options.onConflict ?? "fail";
|
|
10978
11261
|
const useSameServer = plan.sameServer && strategy === "fail" && !plan.crossDialect;
|
|
10979
11262
|
const useCrossDialect = plan.crossDialect && tablePlan.columnTypes;
|
|
10980
|
-
let tableResult
|
|
10981
|
-
let tableErr
|
|
11263
|
+
let tableResult;
|
|
11264
|
+
let tableErr;
|
|
10982
11265
|
if (useSameServer) {
|
|
10983
11266
|
[tableResult, tableErr] = await transferTableSameServer(
|
|
10984
11267
|
ctx,
|
|
@@ -11607,6 +11890,174 @@ var TransferNamespace = class {
|
|
|
11607
11890
|
return getTransferPlan(this.#state.config, destConfig, options);
|
|
11608
11891
|
}
|
|
11609
11892
|
};
|
|
11893
|
+
|
|
11894
|
+
// src/core/worker-bridge/pool.ts
|
|
11895
|
+
var WorkerPool = class {
|
|
11896
|
+
#workers;
|
|
11897
|
+
#nextIndex = 0;
|
|
11898
|
+
#isShutdown = false;
|
|
11899
|
+
constructor(script, options) {
|
|
11900
|
+
const size = Math.max(1, options.size);
|
|
11901
|
+
this.#workers = Array.from(
|
|
11902
|
+
{ length: size },
|
|
11903
|
+
() => new WorkerBridge(script)
|
|
11904
|
+
);
|
|
11905
|
+
}
|
|
11906
|
+
/**
|
|
11907
|
+
* Returns the number of workers in the pool.
|
|
11908
|
+
*/
|
|
11909
|
+
get size() {
|
|
11910
|
+
return this.#workers.length;
|
|
11911
|
+
}
|
|
11912
|
+
/**
|
|
11913
|
+
* Returns true if the pool has been shut down.
|
|
11914
|
+
*/
|
|
11915
|
+
get isShutdown() {
|
|
11916
|
+
return this.#isShutdown;
|
|
11917
|
+
}
|
|
11918
|
+
/**
|
|
11919
|
+
* Sends a request to the next available worker (round-robin) and awaits its response.
|
|
11920
|
+
*
|
|
11921
|
+
* Injects a correlation ID so concurrent requests on the same event type
|
|
11922
|
+
* can be matched to their originating worker.
|
|
11923
|
+
*
|
|
11924
|
+
* @example
|
|
11925
|
+
* const { values } = await pool.request('serialize', { row, columns, index: 0 })
|
|
11926
|
+
*/
|
|
11927
|
+
async request(event, data2, options) {
|
|
11928
|
+
const worker = this.#workers[this.#nextIndex];
|
|
11929
|
+
this.#nextIndex = (this.#nextIndex + 1) % this.#workers.length;
|
|
11930
|
+
return worker.request(event, data2, options);
|
|
11931
|
+
}
|
|
11932
|
+
/**
|
|
11933
|
+
* Registers a listener on all workers for the given event.
|
|
11934
|
+
*
|
|
11935
|
+
* Useful for broadcast-style events that every worker may emit.
|
|
11936
|
+
*
|
|
11937
|
+
* @example
|
|
11938
|
+
* pool.on('serialize:res', ({ data }) => console.log(data.values))
|
|
11939
|
+
*/
|
|
11940
|
+
on(event, callback, options) {
|
|
11941
|
+
for (const worker of this.#workers) {
|
|
11942
|
+
worker.on(event, callback, options);
|
|
11943
|
+
}
|
|
11944
|
+
}
|
|
11945
|
+
/**
|
|
11946
|
+
* Terminates all workers in the pool.
|
|
11947
|
+
*
|
|
11948
|
+
* Marks the pool as shut down and waits for every worker thread to exit
|
|
11949
|
+
* before resolving.
|
|
11950
|
+
*
|
|
11951
|
+
* @example
|
|
11952
|
+
* await pool.shutdown()
|
|
11953
|
+
*/
|
|
11954
|
+
async shutdown() {
|
|
11955
|
+
this.#isShutdown = true;
|
|
11956
|
+
await Promise.all(this.#workers.map((w) => w.shutdown()));
|
|
11957
|
+
}
|
|
11958
|
+
};
|
|
11959
|
+
|
|
11960
|
+
// src/core/worker-bridge/types.ts
|
|
11961
|
+
function isTransferable(data2) {
|
|
11962
|
+
return !!data2 && typeof data2 === "object" && "__transfer" in data2;
|
|
11963
|
+
}
|
|
11964
|
+
|
|
11965
|
+
// src/core/worker-bridge/bridge.ts
|
|
11966
|
+
var WorkerBridge = class extends ObserverRelay {
|
|
11967
|
+
#port;
|
|
11968
|
+
#ownsWorker;
|
|
11969
|
+
constructor(script, data2) {
|
|
11970
|
+
const isParent = !!script;
|
|
11971
|
+
super({ name: isParent ? "bridge:parent" : "bridge:worker" });
|
|
11972
|
+
if (script) {
|
|
11973
|
+
this.#port = new Worker(script, { workerData: data2 });
|
|
11974
|
+
this.#ownsWorker = true;
|
|
11975
|
+
this.#port.on("exit", (code) => {
|
|
11976
|
+
if (!this.isShutdown && code !== 0) {
|
|
11977
|
+
this.receive("worker:exit", { code, error: `Worker exited with code ${code}` }, {});
|
|
11978
|
+
}
|
|
11979
|
+
});
|
|
11980
|
+
} else {
|
|
11981
|
+
if (!parentPort) {
|
|
11982
|
+
throw new Error(
|
|
11983
|
+
"WorkerBridge: no script provided and not in a worker thread"
|
|
11984
|
+
);
|
|
11985
|
+
}
|
|
11986
|
+
this.#port = parentPort;
|
|
11987
|
+
this.#ownsWorker = false;
|
|
11988
|
+
}
|
|
11989
|
+
this.#port.on("message", (msg) => {
|
|
11990
|
+
this.receive(msg.event, msg.data, {});
|
|
11991
|
+
});
|
|
11992
|
+
}
|
|
11993
|
+
send(event, data2) {
|
|
11994
|
+
if (isTransferable(data2)) {
|
|
11995
|
+
this.#port.postMessage({ event, data: data2 }, data2.__transfer);
|
|
11996
|
+
} else {
|
|
11997
|
+
this.#port.postMessage({ event, data: data2 }, []);
|
|
11998
|
+
}
|
|
11999
|
+
}
|
|
12000
|
+
async request(event, data2, options) {
|
|
12001
|
+
const cid = randomUUID();
|
|
12002
|
+
const pending = this.once(new RegExp(`^${event}:res:${cid}$`), options);
|
|
12003
|
+
this.send(event, { ...data2, __cid: cid });
|
|
12004
|
+
const { data: { data: result } } = await pending;
|
|
12005
|
+
return result;
|
|
12006
|
+
}
|
|
12007
|
+
transfer(event, data2, transferables) {
|
|
12008
|
+
this.#port.postMessage({ event, data: data2 }, transferables);
|
|
12009
|
+
}
|
|
12010
|
+
static pool(script, options) {
|
|
12011
|
+
return new WorkerPool(script, options);
|
|
12012
|
+
}
|
|
12013
|
+
static get workerData() {
|
|
12014
|
+
return workerData;
|
|
12015
|
+
}
|
|
12016
|
+
async shutdown() {
|
|
12017
|
+
super.shutdown();
|
|
12018
|
+
if (this.#ownsWorker && "terminate" in this.#port) {
|
|
12019
|
+
await this.#port.terminate();
|
|
12020
|
+
}
|
|
12021
|
+
}
|
|
12022
|
+
};
|
|
12023
|
+
|
|
12024
|
+
// src/core/worker-bridge/order-buffer.ts
|
|
12025
|
+
var OrderBuffer = class {
|
|
12026
|
+
#buffer = /* @__PURE__ */ new Map();
|
|
12027
|
+
#nextIndex = 0;
|
|
12028
|
+
#flush;
|
|
12029
|
+
constructor(flush) {
|
|
12030
|
+
this.#flush = flush;
|
|
12031
|
+
}
|
|
12032
|
+
get nextIndex() {
|
|
12033
|
+
return this.#nextIndex;
|
|
12034
|
+
}
|
|
12035
|
+
get pending() {
|
|
12036
|
+
return this.#buffer.size;
|
|
12037
|
+
}
|
|
12038
|
+
add(index, item) {
|
|
12039
|
+
this.#buffer.set(index, item);
|
|
12040
|
+
this.#drain();
|
|
12041
|
+
}
|
|
12042
|
+
#drain() {
|
|
12043
|
+
while (this.#buffer.has(this.#nextIndex)) {
|
|
12044
|
+
const item = this.#buffer.get(this.#nextIndex);
|
|
12045
|
+
this.#buffer.delete(this.#nextIndex);
|
|
12046
|
+
this.#nextIndex++;
|
|
12047
|
+
this.#flush(item);
|
|
12048
|
+
}
|
|
12049
|
+
}
|
|
12050
|
+
};
|
|
12051
|
+
var IS_COMPILED = import.meta.url.includes("$bunfs");
|
|
12052
|
+
var meta = import.meta;
|
|
12053
|
+
var MODULE_DIR = typeof meta.dir === "string" ? meta.dir : dirname(fileURLToPath(import.meta.url));
|
|
12054
|
+
var WORKER_DIR = resolve(MODULE_DIR, "../../workers");
|
|
12055
|
+
function resolveWorker(name) {
|
|
12056
|
+
if (IS_COMPILED) {
|
|
12057
|
+
return new URL(`./workers/${name}.js`, import.meta.url);
|
|
12058
|
+
}
|
|
12059
|
+
return resolve(WORKER_DIR, `${name}.js`);
|
|
12060
|
+
}
|
|
11610
12061
|
var DtWriter = class {
|
|
11611
12062
|
#filepath;
|
|
11612
12063
|
#schema;
|
|
@@ -11710,8 +12161,8 @@ var DtWriter = class {
|
|
|
11710
12161
|
this.#passthrough.end();
|
|
11711
12162
|
await this.#pipelinePromise;
|
|
11712
12163
|
} else if (this.#stream) {
|
|
11713
|
-
await new Promise((
|
|
11714
|
-
this.#stream.end(() =>
|
|
12164
|
+
await new Promise((resolve2, reject) => {
|
|
12165
|
+
this.#stream.end(() => resolve2());
|
|
11715
12166
|
this.#stream.on("error", reject);
|
|
11716
12167
|
});
|
|
11717
12168
|
}
|
|
@@ -11728,211 +12179,26 @@ var DtWriter = class {
|
|
|
11728
12179
|
this.#stream.write(buf);
|
|
11729
12180
|
}
|
|
11730
12181
|
};
|
|
11731
|
-
|
|
11732
|
-
|
|
11733
|
-
|
|
11734
|
-
|
|
11735
|
-
|
|
11736
|
-
|
|
11737
|
-
|
|
11738
|
-
|
|
11739
|
-
}
|
|
11740
|
-
function serializeValue(value, column) {
|
|
11741
|
-
if (value === null || value === void 0) {
|
|
11742
|
-
return null;
|
|
11743
|
-
}
|
|
11744
|
-
if (isEncodedType(column.type)) {
|
|
11745
|
-
return encodeValue(value, column.type);
|
|
11746
|
-
}
|
|
11747
|
-
return serializeSimple(value, column.type);
|
|
11748
|
-
}
|
|
11749
|
-
function serializeSimple(value, type) {
|
|
11750
|
-
switch (type) {
|
|
11751
|
-
case "bigint":
|
|
11752
|
-
return String(value);
|
|
11753
|
-
case "decimal":
|
|
11754
|
-
return String(value);
|
|
11755
|
-
case "bool":
|
|
11756
|
-
if (typeof value === "number") return value !== 0;
|
|
11757
|
-
if (typeof value === "string") return value === "1" || value.toLowerCase() === "true";
|
|
11758
|
-
return Boolean(value);
|
|
11759
|
-
case "timestamp": {
|
|
11760
|
-
if (value instanceof Date) return value.toISOString();
|
|
11761
|
-
return String(value);
|
|
11762
|
-
}
|
|
11763
|
-
case "date": {
|
|
11764
|
-
if (value instanceof Date) return value.toISOString().split("T")[0];
|
|
11765
|
-
return String(value);
|
|
11766
|
-
}
|
|
11767
|
-
case "uuid":
|
|
11768
|
-
return String(value);
|
|
11769
|
-
case "int":
|
|
11770
|
-
case "float":
|
|
11771
|
-
return typeof value === "string" ? Number(value) : value;
|
|
11772
|
-
case "string":
|
|
11773
|
-
default:
|
|
11774
|
-
return String(value);
|
|
11775
|
-
}
|
|
11776
|
-
}
|
|
11777
|
-
function encodeValue(value, type) {
|
|
11778
|
-
if (type === "binary") {
|
|
11779
|
-
return encodeBinary(value);
|
|
11780
|
-
}
|
|
11781
|
-
return encodeJsonLike(value);
|
|
11782
|
-
}
|
|
11783
|
-
function encodeBinary(value) {
|
|
11784
|
-
const buf = toBuffer(value);
|
|
11785
|
-
const byteLength = buf.length;
|
|
11786
|
-
if (byteLength < GZIP_THRESHOLD) {
|
|
11787
|
-
return [buf.toString("base64"), "b64"];
|
|
11788
|
-
}
|
|
11789
|
-
const compressed = gzipSync(buf);
|
|
11790
|
-
if (compressed.length < byteLength * GZIP_RATIO_THRESHOLD) {
|
|
11791
|
-
return [compressed.toString("base64"), "gz64"];
|
|
11792
|
-
}
|
|
11793
|
-
return [buf.toString("base64"), "b64"];
|
|
11794
|
-
}
|
|
11795
|
-
function encodeJsonLike(value) {
|
|
11796
|
-
const jsonStr = JSON.stringify(value);
|
|
11797
|
-
const byteLength = Buffer.byteLength(jsonStr, "utf8");
|
|
11798
|
-
if (byteLength < GZIP_THRESHOLD) {
|
|
11799
|
-
return [value, "raw"];
|
|
11800
|
-
}
|
|
11801
|
-
const buf = Buffer.from(jsonStr, "utf8");
|
|
11802
|
-
const compressed = gzipSync(buf);
|
|
11803
|
-
if (compressed.length < byteLength * GZIP_RATIO_THRESHOLD) {
|
|
11804
|
-
return [compressed.toString("base64"), "gz64"];
|
|
11805
|
-
}
|
|
11806
|
-
return [value, "raw"];
|
|
11807
|
-
}
|
|
11808
|
-
function toBuffer(value) {
|
|
11809
|
-
if (Buffer.isBuffer(value)) return value;
|
|
11810
|
-
if (value instanceof Uint8Array) return Buffer.from(value);
|
|
11811
|
-
if (typeof value === "string") return Buffer.from(value, "base64");
|
|
11812
|
-
return Buffer.from(String(value), "utf8");
|
|
11813
|
-
}
|
|
11814
|
-
function deserializeRow(options) {
|
|
11815
|
-
const { values, columns, targetDialect, targetVersion } = options;
|
|
11816
|
-
const result = {};
|
|
11817
|
-
for (let i = 0; i < columns.length; i++) {
|
|
11818
|
-
const col = columns[i];
|
|
11819
|
-
const value = values[i];
|
|
11820
|
-
result[col.name] = deserializeValue(value, col, targetDialect, targetVersion);
|
|
11821
|
-
}
|
|
11822
|
-
return result;
|
|
11823
|
-
}
|
|
11824
|
-
function deserializeValue(value, column, targetDialect, targetVersion) {
|
|
11825
|
-
if (value === null || value === void 0) {
|
|
11826
|
-
return null;
|
|
11827
|
-
}
|
|
11828
|
-
if (isEncodedType(column.type)) {
|
|
11829
|
-
const decoded = decodeTuple(value);
|
|
11830
|
-
return convertEncodedForTarget(decoded, column.type, targetDialect, targetVersion);
|
|
11831
|
-
}
|
|
11832
|
-
return convertSimpleForTarget(value, column.type, targetDialect);
|
|
11833
|
-
}
|
|
11834
|
-
function decodeTuple(tuple) {
|
|
11835
|
-
if (!Array.isArray(tuple) || tuple.length < 2) {
|
|
11836
|
-
return tuple;
|
|
11837
|
-
}
|
|
11838
|
-
const [value, encoding] = tuple;
|
|
11839
|
-
switch (encoding) {
|
|
11840
|
-
case "raw":
|
|
11841
|
-
return value;
|
|
11842
|
-
case "b64":
|
|
11843
|
-
return Buffer.from(value, "base64");
|
|
11844
|
-
case "gz64": {
|
|
11845
|
-
const compressed = Buffer.from(value, "base64");
|
|
11846
|
-
const decompressed = gunzipSync(compressed);
|
|
11847
|
-
const str = decompressed.toString("utf8");
|
|
11848
|
-
if (/^[[{"\d]/.test(str)) {
|
|
11849
|
-
return JSON.parse(str);
|
|
11850
|
-
}
|
|
11851
|
-
return decompressed;
|
|
11852
|
-
}
|
|
11853
|
-
default:
|
|
11854
|
-
return value;
|
|
11855
|
-
}
|
|
11856
|
-
}
|
|
11857
|
-
function convertEncodedForTarget(decoded, type, targetDialect, targetVersion) {
|
|
11858
|
-
switch (type) {
|
|
11859
|
-
case "json":
|
|
11860
|
-
return convertJsonForTarget(decoded, targetDialect, targetVersion);
|
|
11861
|
-
case "text":
|
|
11862
|
-
return decoded;
|
|
11863
|
-
case "binary":
|
|
11864
|
-
return decoded;
|
|
11865
|
-
case "vector":
|
|
11866
|
-
return convertVectorForTarget(decoded, targetDialect, targetVersion);
|
|
11867
|
-
case "array":
|
|
11868
|
-
return convertArrayForTarget(decoded, targetDialect);
|
|
11869
|
-
case "custom":
|
|
11870
|
-
if (typeof decoded === "object") return JSON.stringify(decoded);
|
|
11871
|
-
return decoded;
|
|
11872
|
-
default:
|
|
11873
|
-
return decoded;
|
|
11874
|
-
}
|
|
11875
|
-
}
|
|
11876
|
-
function convertJsonForTarget(decoded, targetDialect, targetVersion) {
|
|
11877
|
-
if (targetDialect === "mssql") {
|
|
11878
|
-
const major = targetVersion?.major ?? 2022;
|
|
11879
|
-
if (major < 2025) {
|
|
11880
|
-
return typeof decoded === "string" ? decoded : JSON.stringify(decoded);
|
|
11881
|
-
}
|
|
11882
|
-
}
|
|
11883
|
-
return decoded;
|
|
11884
|
-
}
|
|
11885
|
-
function convertVectorForTarget(decoded, targetDialect, targetVersion) {
|
|
11886
|
-
const arr = Array.isArray(decoded) ? decoded : decoded;
|
|
11887
|
-
if (targetDialect === "postgres") {
|
|
11888
|
-
return Array.isArray(arr) ? `[${arr.join(",")}]` : String(arr);
|
|
11889
|
-
}
|
|
11890
|
-
if (targetDialect === "mysql") {
|
|
11891
|
-
const major = targetVersion?.major ?? 8;
|
|
11892
|
-
if (major >= 9) {
|
|
11893
|
-
return Array.isArray(arr) ? `[${arr.join(",")}]` : String(arr);
|
|
11894
|
-
}
|
|
11895
|
-
return Array.isArray(arr) ? JSON.stringify(arr) : String(arr);
|
|
11896
|
-
}
|
|
11897
|
-
return Array.isArray(arr) ? JSON.stringify(arr) : String(arr);
|
|
11898
|
-
}
|
|
11899
|
-
function convertArrayForTarget(decoded, targetDialect) {
|
|
11900
|
-
if (targetDialect === "postgres") {
|
|
11901
|
-
return decoded;
|
|
11902
|
-
}
|
|
11903
|
-
return Array.isArray(decoded) ? JSON.stringify(decoded) : String(decoded);
|
|
12182
|
+
|
|
12183
|
+
// src/core/dt/index.ts
|
|
12184
|
+
var CONNECTION_WORKER = resolveWorker("connection");
|
|
12185
|
+
var COMPUTE_WORKER = resolveWorker("compute");
|
|
12186
|
+
function createDefaultComputePool() {
|
|
12187
|
+
return WorkerBridge.pool(COMPUTE_WORKER, {
|
|
12188
|
+
size: Math.max(1, availableParallelism() - 2)
|
|
12189
|
+
});
|
|
11904
12190
|
}
|
|
11905
|
-
function
|
|
11906
|
-
|
|
11907
|
-
|
|
11908
|
-
|
|
11909
|
-
|
|
11910
|
-
|
|
11911
|
-
|
|
11912
|
-
|
|
11913
|
-
case "decimal":
|
|
11914
|
-
return value;
|
|
11915
|
-
case "bool":
|
|
11916
|
-
if (targetDialect === "mssql") return value ? 1 : 0;
|
|
11917
|
-
if (targetDialect === "mysql") return value ? 1 : 0;
|
|
11918
|
-
return value;
|
|
11919
|
-
case "timestamp":
|
|
11920
|
-
if (typeof value === "string") return new Date(value);
|
|
11921
|
-
return value;
|
|
11922
|
-
case "date":
|
|
11923
|
-
return value;
|
|
11924
|
-
case "uuid":
|
|
11925
|
-
return value;
|
|
11926
|
-
case "int":
|
|
11927
|
-
case "float":
|
|
11928
|
-
return value;
|
|
11929
|
-
case "string":
|
|
11930
|
-
default:
|
|
11931
|
-
return value;
|
|
12191
|
+
async function createDefaultConnectionBridge(dialect, connectionString) {
|
|
12192
|
+
const bridge = new WorkerBridge(CONNECTION_WORKER);
|
|
12193
|
+
const [, connectErr] = await attempt(
|
|
12194
|
+
() => bridge.request("connect", { dialect, connectionString })
|
|
12195
|
+
);
|
|
12196
|
+
if (connectErr) {
|
|
12197
|
+
await attempt(() => bridge.shutdown());
|
|
12198
|
+
return [null, connectErr];
|
|
11932
12199
|
}
|
|
12200
|
+
return [bridge, null];
|
|
11933
12201
|
}
|
|
11934
|
-
|
|
11935
|
-
// src/core/dt/index.ts
|
|
11936
12202
|
async function exportTable(options) {
|
|
11937
12203
|
const startTime = Date.now();
|
|
11938
12204
|
const { db, dialect, tableName, filepath, passphrase, schema: schemaName } = options;
|
|
@@ -11958,58 +12224,178 @@ async function exportTable(options) {
|
|
|
11958
12224
|
if (openErr) {
|
|
11959
12225
|
return [null, openErr];
|
|
11960
12226
|
}
|
|
11961
|
-
let
|
|
12227
|
+
let ownedConnectionBridge = null;
|
|
12228
|
+
let ownedComputePool = null;
|
|
12229
|
+
let connectionBridge = options.connectionBridge;
|
|
12230
|
+
let computePool = options.computePool;
|
|
12231
|
+
if (!connectionBridge && options.connectionString) {
|
|
12232
|
+
const [bridge, bridgeErr] = await createDefaultConnectionBridge(dialect, options.connectionString);
|
|
12233
|
+
if (bridgeErr) {
|
|
12234
|
+
return [null, bridgeErr];
|
|
12235
|
+
}
|
|
12236
|
+
connectionBridge = bridge;
|
|
12237
|
+
ownedConnectionBridge = bridge;
|
|
12238
|
+
}
|
|
12239
|
+
if (!computePool) {
|
|
12240
|
+
computePool = createDefaultComputePool();
|
|
12241
|
+
ownedComputePool = computePool;
|
|
12242
|
+
}
|
|
12243
|
+
const [result, workerErr] = await exportTableWithWorkers({
|
|
12244
|
+
writer,
|
|
12245
|
+
dtSchema,
|
|
12246
|
+
tableName,
|
|
12247
|
+
filepath,
|
|
12248
|
+
dialect,
|
|
12249
|
+
batchSize,
|
|
12250
|
+
connectionBridge,
|
|
12251
|
+
computePool,
|
|
12252
|
+
kyselyDb
|
|
12253
|
+
});
|
|
12254
|
+
if (ownedComputePool) {
|
|
12255
|
+
await attempt(() => ownedComputePool.shutdown());
|
|
12256
|
+
}
|
|
12257
|
+
if (ownedConnectionBridge) {
|
|
12258
|
+
await attempt(() => ownedConnectionBridge.shutdown());
|
|
12259
|
+
}
|
|
12260
|
+
if (workerErr) {
|
|
12261
|
+
return [null, workerErr];
|
|
12262
|
+
}
|
|
12263
|
+
const durationMs = Date.now() - startTime;
|
|
12264
|
+
observer.emit("dt:export:complete", {
|
|
12265
|
+
filepath,
|
|
12266
|
+
table: tableName,
|
|
12267
|
+
rowsWritten: result.rowsWritten,
|
|
12268
|
+
bytesWritten: result.bytesWritten,
|
|
12269
|
+
durationMs
|
|
12270
|
+
});
|
|
12271
|
+
return [result, null];
|
|
12272
|
+
}
|
|
12273
|
+
function getQuoteIdent(dialect) {
|
|
12274
|
+
if (dialect === "mssql") return (c) => `[${c}]`;
|
|
12275
|
+
if (dialect === "mysql") return (c) => `\`${c}\``;
|
|
12276
|
+
return (c) => `"${c}"`;
|
|
12277
|
+
}
|
|
12278
|
+
function buildBatchSql(dialect, tableName, columnList, orderCol, batchSize, offset) {
|
|
12279
|
+
if (dialect === "mssql") {
|
|
12280
|
+
return `SELECT ${columnList} FROM [${tableName}] ORDER BY ${orderCol} OFFSET ${offset} ROWS FETCH NEXT ${batchSize} ROWS ONLY`;
|
|
12281
|
+
}
|
|
12282
|
+
return `SELECT ${columnList} FROM "${tableName}" LIMIT ${batchSize} OFFSET ${offset}`;
|
|
12283
|
+
}
|
|
12284
|
+
async function exportTableWithWorkers(ctx) {
|
|
12285
|
+
const { writer, dtSchema, tableName, filepath, dialect, batchSize } = ctx;
|
|
12286
|
+
const { connectionBridge, computePool, kyselyDb } = ctx;
|
|
12287
|
+
const backpressureLimit = batchSize * 3;
|
|
12288
|
+
let totalRows = 0;
|
|
12289
|
+
if (connectionBridge) {
|
|
12290
|
+
const countSql = dialect === "mssql" ? `SELECT COUNT(*) AS cnt FROM [${tableName}]` : `SELECT COUNT(*) AS cnt FROM "${tableName}"`;
|
|
12291
|
+
const [countResult, countErr] = await attempt(
|
|
12292
|
+
() => connectionBridge.request("query", { sql: countSql })
|
|
12293
|
+
);
|
|
12294
|
+
if (countErr) {
|
|
12295
|
+
return [null, countErr];
|
|
12296
|
+
}
|
|
12297
|
+
const firstRow = countResult.rows[0];
|
|
12298
|
+
totalRows = Number(firstRow?.["cnt"] ?? 0);
|
|
12299
|
+
}
|
|
11962
12300
|
const columns = dtSchema.columns.map((c) => c.name);
|
|
11963
|
-
const quoteIdent4 =
|
|
12301
|
+
const quoteIdent4 = getQuoteIdent(dialect);
|
|
11964
12302
|
const columnList = columns.map(quoteIdent4).join(", ");
|
|
12303
|
+
const orderCol = quoteIdent4(columns[0]);
|
|
12304
|
+
let globalIndex = 0;
|
|
12305
|
+
let loaded = 0;
|
|
12306
|
+
let processed = 0;
|
|
12307
|
+
let saved = 0;
|
|
12308
|
+
let inFlight = 0;
|
|
12309
|
+
let pipelineError = null;
|
|
12310
|
+
const orderBuffer = new OrderBuffer((values) => {
|
|
12311
|
+
writer.writeRow(values);
|
|
12312
|
+
saved++;
|
|
12313
|
+
inFlight--;
|
|
12314
|
+
observer.emit("dt:export:saved", { table: tableName, saved, totalRows });
|
|
12315
|
+
});
|
|
12316
|
+
let offset = 0;
|
|
11965
12317
|
while (true) {
|
|
11966
|
-
|
|
11967
|
-
|
|
11968
|
-
|
|
12318
|
+
while (inFlight >= backpressureLimit) {
|
|
12319
|
+
await new Promise((resolve2) => setTimeout(resolve2, 1));
|
|
12320
|
+
}
|
|
12321
|
+
let batchRows;
|
|
12322
|
+
if (connectionBridge) {
|
|
12323
|
+
const batchSql = buildBatchSql(dialect, tableName, columnList, orderCol, batchSize, offset);
|
|
12324
|
+
const [queryResult, queryErr] = await attempt(
|
|
12325
|
+
() => connectionBridge.request("query", { sql: batchSql })
|
|
12326
|
+
);
|
|
12327
|
+
if (queryErr) {
|
|
12328
|
+
pipelineError = queryErr;
|
|
12329
|
+
break;
|
|
12330
|
+
}
|
|
12331
|
+
if (queryResult.error) {
|
|
12332
|
+
pipelineError = new Error(queryResult.error);
|
|
12333
|
+
break;
|
|
12334
|
+
}
|
|
12335
|
+
batchRows = queryResult.rows;
|
|
12336
|
+
} else {
|
|
12337
|
+
const [rows, fetchErr] = await attempt(() => {
|
|
12338
|
+
if (dialect === "mssql") {
|
|
12339
|
+
return sql`
|
|
12340
|
+
SELECT ${sql.raw(columnList)}
|
|
12341
|
+
FROM ${sql.table(tableName)}
|
|
12342
|
+
ORDER BY ${sql.raw(orderCol)}
|
|
12343
|
+
OFFSET ${offset} ROWS
|
|
12344
|
+
FETCH NEXT ${batchSize} ROWS ONLY
|
|
12345
|
+
`.execute(kyselyDb);
|
|
12346
|
+
}
|
|
11969
12347
|
return sql`
|
|
11970
12348
|
SELECT ${sql.raw(columnList)}
|
|
11971
12349
|
FROM ${sql.table(tableName)}
|
|
11972
|
-
|
|
11973
|
-
OFFSET ${offset}
|
|
11974
|
-
FETCH NEXT ${batchSize} ROWS ONLY
|
|
12350
|
+
LIMIT ${batchSize}
|
|
12351
|
+
OFFSET ${offset}
|
|
11975
12352
|
`.execute(kyselyDb);
|
|
12353
|
+
});
|
|
12354
|
+
if (fetchErr) {
|
|
12355
|
+
pipelineError = fetchErr;
|
|
12356
|
+
break;
|
|
11976
12357
|
}
|
|
11977
|
-
|
|
11978
|
-
|
|
11979
|
-
|
|
11980
|
-
|
|
11981
|
-
|
|
11982
|
-
|
|
11983
|
-
|
|
11984
|
-
|
|
11985
|
-
|
|
11986
|
-
|
|
11987
|
-
|
|
11988
|
-
|
|
11989
|
-
|
|
11990
|
-
|
|
12358
|
+
batchRows = rows.rows;
|
|
12359
|
+
}
|
|
12360
|
+
if (batchRows.length === 0) break;
|
|
12361
|
+
loaded += batchRows.length;
|
|
12362
|
+
observer.emit("dt:export:loaded", { table: tableName, loaded, totalRows });
|
|
12363
|
+
for (const row of batchRows) {
|
|
12364
|
+
const rowIndex = globalIndex++;
|
|
12365
|
+
inFlight++;
|
|
12366
|
+
computePool.request("serialize", {
|
|
12367
|
+
row,
|
|
12368
|
+
columns: dtSchema.columns,
|
|
12369
|
+
index: rowIndex
|
|
12370
|
+
}).then((result) => {
|
|
12371
|
+
processed++;
|
|
12372
|
+
observer.emit("dt:export:processed", { table: tableName, processed, totalRows });
|
|
12373
|
+
if (result.error) {
|
|
12374
|
+
pipelineError = pipelineError ?? new Error(result.error);
|
|
12375
|
+
return;
|
|
12376
|
+
}
|
|
12377
|
+
orderBuffer.add(result.index, result.values);
|
|
12378
|
+
});
|
|
11991
12379
|
}
|
|
11992
|
-
offset +=
|
|
12380
|
+
offset += batchRows.length;
|
|
11993
12381
|
observer.emit("dt:export:progress", {
|
|
11994
12382
|
filepath,
|
|
11995
12383
|
table: tableName,
|
|
11996
12384
|
rowsWritten: writer.rowsWritten,
|
|
11997
12385
|
bytesWritten: writer.bytesWritten
|
|
11998
12386
|
});
|
|
11999
|
-
if (
|
|
12387
|
+
if (batchRows.length < batchSize) break;
|
|
12388
|
+
}
|
|
12389
|
+
while (inFlight > 0) {
|
|
12390
|
+
await new Promise((resolve2) => setTimeout(resolve2, 1));
|
|
12391
|
+
}
|
|
12392
|
+
if (pipelineError) {
|
|
12393
|
+
return [null, pipelineError];
|
|
12000
12394
|
}
|
|
12001
12395
|
const [, closeErr] = await attempt(() => writer.close());
|
|
12002
12396
|
if (closeErr) {
|
|
12003
12397
|
return [null, closeErr];
|
|
12004
12398
|
}
|
|
12005
|
-
const durationMs = Date.now() - startTime;
|
|
12006
|
-
observer.emit("dt:export:complete", {
|
|
12007
|
-
filepath,
|
|
12008
|
-
table: tableName,
|
|
12009
|
-
rowsWritten: writer.rowsWritten,
|
|
12010
|
-
bytesWritten: writer.bytesWritten,
|
|
12011
|
-
durationMs
|
|
12012
|
-
});
|
|
12013
12399
|
return [{ rowsWritten: writer.rowsWritten, bytesWritten: writer.bytesWritten }, null];
|
|
12014
12400
|
}
|
|
12015
12401
|
async function importDtFile(options) {
|
|
@@ -12069,70 +12455,158 @@ async function importDtFile(options) {
|
|
|
12069
12455
|
return [null, err];
|
|
12070
12456
|
}
|
|
12071
12457
|
}
|
|
12458
|
+
let ownedComputePool = null;
|
|
12459
|
+
let computePool = options.computePool;
|
|
12460
|
+
if (!computePool) {
|
|
12461
|
+
computePool = createDefaultComputePool();
|
|
12462
|
+
ownedComputePool = computePool;
|
|
12463
|
+
}
|
|
12464
|
+
const [result, workerErr] = await importFileWithWorkers({
|
|
12465
|
+
reader,
|
|
12466
|
+
dtSchema,
|
|
12467
|
+
tableName,
|
|
12468
|
+
filepath,
|
|
12469
|
+
dialect,
|
|
12470
|
+
version,
|
|
12471
|
+
batchSize,
|
|
12472
|
+
onConflict: options.onConflict ?? "fail",
|
|
12473
|
+
computePool,
|
|
12474
|
+
kyselyDb
|
|
12475
|
+
});
|
|
12476
|
+
if (ownedComputePool) {
|
|
12477
|
+
await attempt(() => ownedComputePool.shutdown());
|
|
12478
|
+
}
|
|
12479
|
+
if (workerErr) {
|
|
12480
|
+
reader.close();
|
|
12481
|
+
observer.emit("error", { source: "dt:import", error: workerErr, context: { filepath, table: tableName } });
|
|
12482
|
+
return [null, workerErr];
|
|
12483
|
+
}
|
|
12484
|
+
reader.close();
|
|
12485
|
+
const durationMs = Date.now() - startTime;
|
|
12486
|
+
observer.emit("dt:import:complete", {
|
|
12487
|
+
filepath,
|
|
12488
|
+
table: tableName,
|
|
12489
|
+
rowsImported: result.rowsImported,
|
|
12490
|
+
rowsSkipped: result.rowsSkipped,
|
|
12491
|
+
durationMs
|
|
12492
|
+
});
|
|
12493
|
+
return [result, null];
|
|
12494
|
+
}
|
|
12495
|
+
async function importFileWithWorkers(ctx) {
|
|
12496
|
+
const { reader, dtSchema, tableName, filepath, dialect, version, batchSize } = ctx;
|
|
12497
|
+
const { onConflict, computePool, kyselyDb } = ctx;
|
|
12072
12498
|
let rowsImported = 0;
|
|
12073
12499
|
let rowsSkipped = 0;
|
|
12074
|
-
let
|
|
12075
|
-
|
|
12076
|
-
|
|
12077
|
-
|
|
12078
|
-
|
|
12079
|
-
|
|
12080
|
-
|
|
12081
|
-
|
|
12082
|
-
|
|
12083
|
-
|
|
12084
|
-
|
|
12085
|
-
const [batchResult, insertErr] = await insertImportBatch(
|
|
12086
|
-
kyselyDb,
|
|
12087
|
-
tableName,
|
|
12088
|
-
batch,
|
|
12089
|
-
options.onConflict ?? "fail",
|
|
12090
|
-
dialect,
|
|
12091
|
-
columnNames
|
|
12092
|
-
);
|
|
12093
|
-
if (insertErr) {
|
|
12094
|
-
reader.close();
|
|
12095
|
-
observer.emit("error", { source: "dt:import", error: insertErr, context: { filepath, table: tableName } });
|
|
12096
|
-
return [null, insertErr];
|
|
12097
|
-
}
|
|
12098
|
-
rowsImported += batchResult.inserted + batchResult.updated;
|
|
12099
|
-
rowsSkipped += batchResult.skipped;
|
|
12100
|
-
batch = [];
|
|
12101
|
-
observer.emit("dt:import:progress", {
|
|
12102
|
-
filepath,
|
|
12103
|
-
table: tableName,
|
|
12104
|
-
rowsImported,
|
|
12105
|
-
rowsSkipped
|
|
12106
|
-
});
|
|
12107
|
-
}
|
|
12108
|
-
}
|
|
12109
|
-
if (batch.length > 0) {
|
|
12110
|
-
const columnNames = dtSchema.columns.map((c) => c.name);
|
|
12500
|
+
let loaded = 0;
|
|
12501
|
+
let processed = 0;
|
|
12502
|
+
let saved = 0;
|
|
12503
|
+
let globalIndex = 0;
|
|
12504
|
+
let inFlight = 0;
|
|
12505
|
+
let pipelineError = null;
|
|
12506
|
+
let totalRows = 0;
|
|
12507
|
+
let insertBatch2 = [];
|
|
12508
|
+
const columnNames = dtSchema.columns.map((c) => c.name);
|
|
12509
|
+
const flushInsertBatch = async () => {
|
|
12510
|
+
if (insertBatch2.length === 0) return null;
|
|
12111
12511
|
const [batchResult, insertErr] = await insertImportBatch(
|
|
12112
12512
|
kyselyDb,
|
|
12113
12513
|
tableName,
|
|
12114
|
-
|
|
12115
|
-
|
|
12514
|
+
insertBatch2,
|
|
12515
|
+
onConflict,
|
|
12116
12516
|
dialect,
|
|
12117
12517
|
columnNames
|
|
12118
12518
|
);
|
|
12119
12519
|
if (insertErr) {
|
|
12120
|
-
|
|
12121
|
-
observer.emit("error", { source: "dt:import", error: insertErr, context: { filepath, table: tableName } });
|
|
12122
|
-
return [null, insertErr];
|
|
12520
|
+
return insertErr;
|
|
12123
12521
|
}
|
|
12124
12522
|
rowsImported += batchResult.inserted + batchResult.updated;
|
|
12125
12523
|
rowsSkipped += batchResult.skipped;
|
|
12126
|
-
|
|
12127
|
-
|
|
12128
|
-
|
|
12129
|
-
|
|
12130
|
-
|
|
12131
|
-
|
|
12132
|
-
|
|
12133
|
-
|
|
12134
|
-
|
|
12524
|
+
insertBatch2 = [];
|
|
12525
|
+
observer.emit("dt:import:progress", {
|
|
12526
|
+
filepath,
|
|
12527
|
+
table: tableName,
|
|
12528
|
+
rowsImported,
|
|
12529
|
+
rowsSkipped
|
|
12530
|
+
});
|
|
12531
|
+
return null;
|
|
12532
|
+
};
|
|
12533
|
+
const orderBuffer = new OrderBuffer((record) => {
|
|
12534
|
+
insertBatch2.push(record);
|
|
12535
|
+
saved++;
|
|
12536
|
+
inFlight--;
|
|
12537
|
+
observer.emit("dt:import:saved", { table: tableName, saved, totalRows });
|
|
12135
12538
|
});
|
|
12539
|
+
let readBatch = [];
|
|
12540
|
+
for await (const values of reader.rows()) {
|
|
12541
|
+
readBatch.push(values);
|
|
12542
|
+
loaded++;
|
|
12543
|
+
totalRows = Math.max(totalRows, loaded);
|
|
12544
|
+
if (readBatch.length >= batchSize) {
|
|
12545
|
+
observer.emit("dt:import:loaded", { table: tableName, loaded, totalRows });
|
|
12546
|
+
for (const rowValues of readBatch) {
|
|
12547
|
+
const rowIndex = globalIndex++;
|
|
12548
|
+
inFlight++;
|
|
12549
|
+
computePool.request("deserialize", {
|
|
12550
|
+
values: rowValues,
|
|
12551
|
+
columns: dtSchema.columns,
|
|
12552
|
+
targetDialect: dialect,
|
|
12553
|
+
targetVersion: version ? `${version.major}.${version.minor}` : void 0,
|
|
12554
|
+
index: rowIndex
|
|
12555
|
+
}).then((result) => {
|
|
12556
|
+
processed++;
|
|
12557
|
+
observer.emit("dt:import:processed", { table: tableName, processed, totalRows });
|
|
12558
|
+
if (result.error) {
|
|
12559
|
+
pipelineError = pipelineError ?? new Error(result.error);
|
|
12560
|
+
return;
|
|
12561
|
+
}
|
|
12562
|
+
orderBuffer.add(result.index, result.record);
|
|
12563
|
+
});
|
|
12564
|
+
}
|
|
12565
|
+
readBatch = [];
|
|
12566
|
+
while (inFlight > 0) {
|
|
12567
|
+
await new Promise((resolve2) => setTimeout(resolve2, 1));
|
|
12568
|
+
}
|
|
12569
|
+
if (pipelineError) {
|
|
12570
|
+
return [null, pipelineError];
|
|
12571
|
+
}
|
|
12572
|
+
const flushErr = await flushInsertBatch();
|
|
12573
|
+
if (flushErr) {
|
|
12574
|
+
return [null, flushErr];
|
|
12575
|
+
}
|
|
12576
|
+
}
|
|
12577
|
+
}
|
|
12578
|
+
if (readBatch.length > 0) {
|
|
12579
|
+
observer.emit("dt:import:loaded", { table: tableName, loaded, totalRows });
|
|
12580
|
+
for (const rowValues of readBatch) {
|
|
12581
|
+
const rowIndex = globalIndex++;
|
|
12582
|
+
inFlight++;
|
|
12583
|
+
computePool.request("deserialize", {
|
|
12584
|
+
values: rowValues,
|
|
12585
|
+
columns: dtSchema.columns,
|
|
12586
|
+
targetDialect: dialect,
|
|
12587
|
+
targetVersion: version ? `${version.major}.${version.minor}` : void 0,
|
|
12588
|
+
index: rowIndex
|
|
12589
|
+
}).then((result) => {
|
|
12590
|
+
processed++;
|
|
12591
|
+
observer.emit("dt:import:processed", { table: tableName, processed, totalRows });
|
|
12592
|
+
if (result.error) {
|
|
12593
|
+
pipelineError = pipelineError ?? new Error(result.error);
|
|
12594
|
+
return;
|
|
12595
|
+
}
|
|
12596
|
+
orderBuffer.add(result.index, result.record);
|
|
12597
|
+
});
|
|
12598
|
+
}
|
|
12599
|
+
while (inFlight > 0) {
|
|
12600
|
+
await new Promise((resolve2) => setTimeout(resolve2, 1));
|
|
12601
|
+
}
|
|
12602
|
+
if (pipelineError) {
|
|
12603
|
+
return [null, pipelineError];
|
|
12604
|
+
}
|
|
12605
|
+
const flushErr = await flushInsertBatch();
|
|
12606
|
+
if (flushErr) {
|
|
12607
|
+
return [null, flushErr];
|
|
12608
|
+
}
|
|
12609
|
+
}
|
|
12136
12610
|
return [{ rowsImported, rowsSkipped }, null];
|
|
12137
12611
|
}
|
|
12138
12612
|
async function insertImportBatch(db, table, rows, onConflict, dialect, columns) {
|