dsh-db-tool 1.5.91 → 1.7.0

package/dist/store/transfer-history.js

@@ -0,0 +1,114 @@
+/**
+ * transfer-log.jsonl：传输过程日志（追加式，逐行一条事件）。
+ * 事件类型：
+ *  - start    任务开始（statement 含源→目标/定位/模式）
+ *  - table    单表状态变化（开始传输 / 完成 / 失败，含已写入行数）
+ *  - progress 任务进度节流行（运行中各表实时行数，默认 5s 一条）
+ *  - finish   任务终态（全量表明细 + 失败清单；「传输历史」弹窗即取此类型）
+ * 追加写天然原子；损坏行跳过不影响其余。文件位于 <我的文档>/DSH/，用户可直接打开查看。
+ */
+import * as childProcess from 'node:child_process';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+let cachedDocsDir;
+/** 检测「我的文档」真实位置（每台机器可能重定向到任意盘/目录） */
+function detectDocumentsDir() {
+    if (cachedDocsDir)
+        return cachedDocsDir;
+    let dir = '';
+    // Windows 官方途径：Shell API GetFolderPath('MyDocuments')——自动处理文档重定向。
+    // 不用 reg.exe：其中文系统输出为 GBK，utf8 解码会把重定向目录名变成乱码（实测踩坑）
+    if (process.platform === 'win32') {
+        try {
+            const out = childProcess.execFileSync('powershell', ['-NoProfile', '-Command', "[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; [Environment]::GetFolderPath('MyDocuments')"], { timeout: 15000, encoding: 'utf8' });
+            dir = out.trim();
+        }
+        catch {
+            // PowerShell 不可用：走候选路径回退
+        }
+    }
+    if (dir === '') {
+        const home = os.homedir();
+        const candidates = process.platform === 'win32'
+            ? [process.env.OneDrive ? path.join(process.env.OneDrive, 'Documents') : '', path.join(home, 'Documents')]
+            : [process.env.XDG_DOCUMENTS_DIR ?? '', path.join(home, 'Documents')];
+        dir = candidates.find((c) => c !== '' && fs.existsSync(c)) ?? path.join(home, 'Documents');
+    }
+    cachedDocsDir = dir;
+    return dir;
+}
+/** 传输日志存储目录：<我的文档>/DSH/；DSH_TRANSFER_DIR 环境变量可覆盖 */
+export function transferLogDir() {
+    const override = process.env.DSH_TRANSFER_DIR;
+    if (override && override !== '') {
+        fs.mkdirSync(override, { recursive: true });
+        return override;
+    }
+    const dir = path.join(detectDocumentsDir(), 'DSH');
+    fs.mkdirSync(dir, { recursive: true });
+    return dir;
+}
+export class TransferLog {
+    filePath;
+    /** JSONL 文件绝对路径（供「在本机查看」） */
+    get file() {
+        return this.filePath;
+    }
+    constructor(dir) {
+        this.filePath = path.join(dir, 'transfer-log.jsonl');
+        fs.mkdirSync(dir, { recursive: true }); // 目录不存在时 appendFileSync 会 ENOENT
+    }
+    /** 追加一条日志；ts 省略时自动填充当前 UTC 时间 */
+    append(entry) {
+        const full = { ...entry, ts: entry.ts ?? new Date().toISOString() };
+        fs.appendFileSync(this.filePath, JSON.stringify(full) + '\n', 'utf8');
+        return full;
+    }
+    /** 最近 n 条（时间正序）；projectKey/type 给定时过滤。损坏行跳过 */
+    tail(n, projectKey, type) {
+        if (!fs.existsSync(this.filePath))
+            return [];
+        const lines = fs.readFileSync(this.filePath, 'utf8').split('\n');
+        const out = [];
+        for (let i = lines.length - 1; i >= 0 && out.length < n; i--) {
+            const line = lines[i]?.trim();
+            if (!line)
+                continue;
+            try {
+                const e = JSON.parse(line);
+                if (projectKey && e.projectPathKey !== projectKey)
+                    continue;
+                if (type && e.type !== type)
+                    continue;
+                out.push(e);
+            }
+            catch {
+                // 跳过损坏行
+            }
+        }
+        return out.reverse();
+    }
+    /** 清除全部日志。返回删除条数 */
+    clear() {
+        if (!fs.existsSync(this.filePath))
+            return 0;
+        const lines = fs.readFileSync(this.filePath, 'utf8').split('\n');
+        const kept = [];
+        let removed = 0;
+        for (const line of lines) {
+            const t = line.trim();
+            if (!t)
+                continue;
+            try {
+                JSON.parse(t);
+                removed += 1;
+            }
+            catch {
+                kept.push(t); // 损坏行原样保留
+            }
+        }
+        fs.writeFileSync(this.filePath, kept.length > 0 ? kept.join('\n') + '\n' : '', 'utf8');
+        return removed;
+    }
+}

package/dist/transfer.js

@@ -0,0 +1,287 @@
+import { createTransferReader, createTransferWriter, } from './adapters/transfer/model.js';
+// 默认值按各驱动官方批量实践取：oracledb executeMany 防 DPI-1015 需分批（批 ~1000）、
+// mongo bulkWrite 由驱动按 maxWriteBatchSize 自动切批（客户端批大小不敏感）、
+// mysql2/pg 逐行 execute+事务包裹（批大小只影响事务粒度）——批 1000 为全库安全交集
+const DEFAULT_BATCH = 1000;
+const DEFAULT_TABLE_CONCURRENCY = 4;
+const DEFAULT_SHARD_CONCURRENCY = 4;
+/** 单表分片数上限（切块探测查询随之有界） */
+const MAX_SHARDS = 32;
+/** 单批行数上限：readBatch 单批整批进内存（HTTP 可传任意大值打内存） */
+const MAX_BATCH = 5000;
+/** 并行度上限：每 worker 独立 reader/writer 连接池，无界并发会打爆两端（前端表单 max 同值）；
+ *  ponytail: 连接预算 ≈ tableConcurrency × 目标端单池 max（pg 4/mysql 10），目标库 max_connections 偏小时由用户自行调低 */
+const MAX_TABLE_CONCURRENCY = 16;
+const MAX_SHARD_CONCURRENCY = 32;
+/** 失败分片重试次数 */
+const SHARD_RETRIES = 1;
+let seq = 0;
+/** 目标端单写者（SQLite 同库并发写锁死）→ 表级/分片全部串行 */
+function isSingleWriterTarget(kind) {
+    return kind === 'sqlite';
+}
+/** 源端读走专用单连接（oralike）→ 表内分片并行不可用 */
+function supportsShardRead(kind) {
+    return kind !== 'oracle' && kind !== 'dmdb';
+}
+export function startTransferTask(source, target, opts) {
+    const id = `t${Date.now().toString(36)}-${++seq}`;
+    const batchSize = Math.min(MAX_BATCH, Math.max(1, Math.floor(opts.batchSize ?? DEFAULT_BATCH)));
+    // 目标端单写者（SQLite 同库多连接并发写锁死）→ 表级并行钳 1；
+    // 表内分片仍可并行：分片共享同一 writer，其事务块为同步原子段，写入天然串行化
+    const tableConcurrency = isSingleWriterTarget(target.meta.kind)
+        ? 1
+        : Math.min(MAX_TABLE_CONCURRENCY, Math.max(1, Math.floor(opts.tableConcurrency ?? DEFAULT_TABLE_CONCURRENCY)));
+    const shardConcurrency = Math.min(MAX_SHARD_CONCURRENCY, Math.max(1, Math.floor(opts.shardConcurrency ?? DEFAULT_SHARD_CONCURRENCY)));
+    // 定位组装（全缺省 = undefined，读写端走现状路径）；定位仅影响工厂入口，
+    // 分片/进度/快照编排不感知
+    const srcLoc = (opts.sourceDatabase ?? opts.sourceSchema)
+        ? { database: opts.sourceDatabase, schema: opts.sourceSchema }
+        : undefined;
+    const dstLoc = (opts.targetDatabase ?? opts.targetSchema)
+        ? { database: opts.targetDatabase, schema: opts.targetSchema }
+        : undefined;
+    const tables = opts.tables.map((name) => ({
+        name,
+        status: 'pending',
+        rows: 0,
+        totalRows: null,
+        shardsTotal: 0,
+        shardsDone: 0,
+        shardsFailed: 0,
+    }));
+    const failures = [];
+    let status = 'running';
+    let cancelled = false;
+    let resolveFinished;
+    const finished = new Promise((resolve) => {
+        resolveFinished = resolve;
+    });
+    const snapshot = () => ({
+        status,
+        tables: tables.map((t) => ({ ...t })),
+        failures: failures.map((f) => ({ ...f })),
+    });
+    const emit = (ev) => {
+        if (!opts.onLog)
+            return;
+        opts.onLog({ taskId: id, ...ev });
+    };
+    const finish = () => {
+        if (status === 'running')
+            status = cancelled ? 'cancelled' : failures.length > 0 ? 'failed' : 'done';
+        emit({ type: 'finish', status, tables: tables.map((t) => ({ ...t })), failures: failures.map((f) => ({ ...f })) });
+        resolveFinished?.(snapshot());
+    };
+    /** 分片切块：按 batchSize 采样边界键 → [lo, hi) 区间清单（探测查询 ≤ 分片数） */
+    async function planShards(reader, table, column, totalRows) {
+        const shardCount = Math.min(Math.max(1, Math.ceil(totalRows / batchSize)), MAX_SHARDS);
+        if (shardCount <= 1 || !reader.keyAt)
+            return [{ lo: null, hi: null, index: 0 }];
+        const bounds = [];
+        let prev = null;
+        for (let i = 1; i < shardCount; i++) {
+            if (cancelled)
+                break;
+            const key = await reader.keyAt(table, column, i * batchSize);
+            if (key === null)
+                break; // 边界越界（实际行数少于估计）：提前收口
+            if (prev !== null && String(key) === String(prev))
+                continue; // 键重复（同值多行）：跳过空区间
+            bounds.push({ lo: prev, hi: key, index: bounds.length });
+            prev = key;
+        }
+        bounds.push({ lo: bounds.length > 0 ? prev : null, hi: null, index: bounds.length });
+        return bounds;
+    }
+    async function runShard(reader, writer, t, column, shard, prog) {
+        const keyIdx = Math.max(0, t.columns.findIndex((c) => c.name === column));
+        const attempt = async () => {
+            let lo = shard.lo;
+            let loInclusive = true;
+            for (;;) {
+                if (cancelled)
+                    return false; // 取消中断：与「完整跑完」区分（表级 status 依赖此判定）
+                const rows = await reader.readBatch(t.name, { mode: 'range', column, lo, loInclusive, hi: shard.hi }, batchSize);
+                if (rows.length === 0)
+                    return true;
+                // 两步累加：`+= await` 的读改写跨 await，分片并发共享 prog 时会丢其他分片的增量
+                const written = await writer.writeBatch(t, rows);
+                prog.rows += written;
+                if (rows.length < batchSize)
+                    return true;
+                const nextLo = rows[rows.length - 1]?.[keyIdx] ?? null;
+                // 守恒断言：批间链式（k > lo）下 lastKey 不前进 = 结构漂移致列错位，恒取同一批 → 死循环
+                if (!loInclusive && lo !== null && nextLo !== null && String(nextLo) === String(lo)) {
+                    throw new Error(`keyset 分片无进展（键 ${String(lo)}），已中止：源表结构可能在传输期间被变更`);
+                }
+                lo = nextLo;
+                loInclusive = false; // 批间链式：k > lastKey 且 k < hi
+            }
+        };
+        try {
+            return await attempt();
+        }
+        catch (e) {
+            if (cancelled)
+                throw e;
+            return await attempt(); // 分片级重试 1 次（幂等性依赖 writeMode=ignore/replace）
+        }
+    }
+    async function runTable(name, prog) {
+        // reader/writer 创建也在 try 内：单表连接失败只毁该表（进 failures），
+        // 不冒泡终止整个任务；writer 失败时已打开的 reader 由 finally 关闭
+        let reader;
+        let writer;
+        try {
+            prog.status = 'running';
+            emit({ type: 'table', status: 'running', table: name, rows: 0, totalRows: null });
+            emit({ type: 'table', status: 'running', table: name, rows: 0, totalRows: null });
+            const r = await createTransferReader(source, srcLoc);
+            reader = r; // 立即回填外层引用：writer 创建抛错时 finally 仍能关闭已打开的 reader
+            const w = await createTransferWriter(target, opts.writeMode, dstLoc);
+            writer = w;
+            const t = await r.describe(name);
+            prog.totalRows = await r.count(name);
+            // 先查目标表是否存在——有无表都不该「建表失败」：
+            // 无表 → 建表再写；有表 → 按写入模式处理（结构冲突可开覆盖表结构选项）
+            const exists = await w.tableExists(name);
+            if (exists && opts.overwriteStructure) {
+                // 覆盖表结构：删除重建（危险选项；新表结构与源一致，无需清空）
+                await w.dropTable(name);
+                await w.createTable(t);
+            }
+            else if (exists) {
+                if (opts.writeMode === 'truncate' && w.beforeLoad) {
+                    await w.beforeLoad(t); // 清空旧数据再写
+                }
+                else if (opts.writeMode === 'insert') {
+                    throw new Error(`目标表已存在: ${name}（insert 不覆盖已有数据，请改用 ignore/replace/truncate 或开启覆盖表结构）`);
+                }
+                // ignore/replace：写入语句自带冲突处理，直接写
+            }
+            else {
+                await w.createTable(t);
+            }
+            const keyType = await r.shardKey(name);
+            const keyCol = keyType ? (t.columns.find((c) => c.primaryKey) ?? t.columns[0]) : undefined;
+            const shardable = keyType !== null &&
+                keyCol !== undefined &&
+                shardConcurrency > 1 &&
+                supportsShardRead(source.meta.kind) &&
+                r.keyAt !== undefined &&
+                (prog.totalRows ?? 0) > batchSize * 2;
+            if (shardable && keyCol) {
+                const shards = await planShards(r, name, keyCol.name, prog.totalRows ?? 0);
+                prog.shardsTotal = shards.length;
+                const workers = Array.from({ length: Math.min(shardConcurrency, shards.length) }, async () => {
+                    for (;;) {
+                        const shard = shards.shift();
+                        if (shard === undefined || cancelled)
+                            return;
+                        try {
+                            // 返回值区分「完整跑完」与「取消中断」：后者不计入 shardsDone（表级不得标 done）
+                            if (await runShard(r, w, t, keyCol.name, shard, prog))
+                                prog.shardsDone += 1;
+                        }
+                        catch (e) {
+                            prog.shardsFailed += 1;
+                            failures.push({
+                                table: name,
+                                shard: shard.index,
+                                error: e instanceof Error ? e.message : String(e),
+                                lastKey: shard.lo === null ? undefined : String(shard.lo),
+                            });
+                        }
+                    }
+                });
+                await Promise.all(workers);
+            }
+            else {
+                // 顺序链：单分片 range 全序（含 keyset 链语义）；无分片键走 offset
+                prog.shardsTotal = 1;
+                try {
+                    const complete = keyType !== null && keyCol
+                        ? await runShard(r, w, t, keyCol.name, { lo: null, hi: null }, prog)
+                        : await runOffset(r, w, t, prog);
+                    if (complete)
+                        prog.shardsDone += 1;
+                }
+                catch (e) {
+                    prog.shardsFailed += 1;
+                    failures.push({ table: name, shard: -1, error: e instanceof Error ? e.message : String(e) });
+                }
+            }
+            // 表级终态：分片未跑完（取消时 worker 提前退出）不得标 done——否则被误读为「已完整传输」
+            const complete = prog.shardsDone + prog.shardsFailed >= prog.shardsTotal;
+            prog.status = complete
+                ? (prog.shardsFailed > 0 ? 'failed' : 'done')
+                : 'cancelled';
+            emit({ type: 'table', status: prog.status, table: name, rows: prog.rows, totalRows: prog.totalRows });
+        }
+        catch (e) {
+            prog.status = 'failed';
+            prog.error = e instanceof Error ? e.message : String(e);
+            failures.push({ table: name, shard: -1, error: prog.error });
+        }
+        finally {
+            await reader?.close().catch(() => { });
+            await writer?.close().catch(() => { });
+        }
+    }
+    /** 无分片键降级：offset 全序（sqlite 无主键表不用 rowid——评审已证伪该路径） */
+    async function runOffset(reader, writer, t, prog) {
+        for (let offset = 0;; offset += batchSize) {
+            if (cancelled)
+                return false; // 取消中断：与「完整跑完」区分（表级 status 依赖此判定）
+            const rows = await reader.readBatch(t.name, { mode: 'offset', offset }, batchSize);
+            if (rows.length === 0)
+                return true;
+            const written = await writer.writeBatch(t, rows);
+            prog.rows += written;
+            if (rows.length < batchSize)
+                return true;
+        }
+    }
+    // 主流程：表级 worker 池（表间并行；表内分片并行在 runTable 内部）
+    void Promise.resolve().then(async () => {
+        const queue = [...opts.tables];
+        const workers = Array.from({ length: Math.min(tableConcurrency, Math.max(queue.length, 1)) }, async () => {
+            for (;;) {
+                const name = queue.shift();
+                if (name === undefined || cancelled)
+                    return;
+                const prog = tables.find((t) => t.name === name);
+                if (prog === undefined)
+                    continue;
+                await runTable(name, prog);
+            }
+        });
+        try {
+            await Promise.all(workers);
+            finish();
+        }
+        catch (e) {
+            // 兜底（runTable 理论上全捕获）：终态前先等残余 worker 收尾（writer close 在其
+            // finally），否则快照已终态而其他表的写入仍在向目标库进行——同目标互斥按 status
+            // 判定，会让新任务与残余写入并发
+            cancelled = true;
+            await Promise.allSettled(workers);
+            status = 'failed';
+            failures.push({ table: '*', shard: -1, error: e instanceof Error ? e.message : String(e) });
+            finish();
+        }
+    }).catch((e) => {
+        status = 'failed';
+        failures.push({ table: '*', shard: -1, error: e instanceof Error ? e.message : String(e) });
+        finish();
+    });
+    return {
+        id,
+        snapshot,
+        cancel: () => {
+            cancelled = true;
+        },
+        finished,
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-db-tool",
-  "version": "1.5.91",
+  "version": "1.7.0",
   "description": "DSH community plugin: chat-operated multi-database admin tool with sidebar management, project-scoped grants, ro/rw modes and dangerous-operation confirmation",
   "type": "module",
   "license": "MIT",