@toiroakr/lines-db 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/bin/cli.js +105 -2
- package/dist/index.d.cts.map +1 -1
- package/package.json +1 -1
- package/src/cli.ts +336 -152
package/CHANGELOG.md
CHANGED
package/bin/cli.js
CHANGED
|
@@ -1423,7 +1423,110 @@ program.command("validate").description("Validate JSONL file(s) against schema")
|
|
|
1423
1423
|
process.exit(1);
|
|
1424
1424
|
}
|
|
1425
1425
|
});
|
|
1426
|
-
program.command("migrate").description("Migrate data with transformation function").argument("<
|
|
1426
|
+
program.command("migrate").description("Migrate data with transformation function").argument("<path>", "File or directory path to migrate").argument("<transform>", "Transform function (e.g., \"(row) => ({ ...row, age: row.age + 1 })\")").option("-f, --filter <expr>", "Filter expression").option("-e, --errorOutput <path>", "Output file path for transformed data when migration fails").option("-v, --verbose", "Show verbose error output", false).action(async (path, transformStr, options) => {
|
|
1427
|
+
try {
|
|
1428
|
+
const stats = await stat(path);
|
|
1429
|
+
if (stats.isDirectory()) await migrateDirectory(path, transformStr, options);
|
|
1430
|
+
else if (stats.isFile() && path.endsWith(".jsonl")) await migrateFile(path, transformStr, options);
|
|
1431
|
+
else {
|
|
1432
|
+
console.error(`Error: Invalid path: ${path}. Must be a directory or .jsonl file.`);
|
|
1433
|
+
process.exit(1);
|
|
1434
|
+
}
|
|
1435
|
+
} catch (error) {
|
|
1436
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT") console.error(`Error: Path not found: ${path}`);
|
|
1437
|
+
else console.error(`Error: ${String(error)}`);
|
|
1438
|
+
process.exit(1);
|
|
1439
|
+
}
|
|
1440
|
+
});
|
|
1441
|
+
/**
|
|
1442
|
+
* Migrate all JSONL files in a directory
|
|
1443
|
+
*/
|
|
1444
|
+
async function migrateDirectory(dirPath, transformStr, options) {
|
|
1445
|
+
const jsonlFiles = (await readdir(dirPath, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => entry.name);
|
|
1446
|
+
if (jsonlFiles.length === 0) {
|
|
1447
|
+
console.error(`Error: No JSONL files found in directory: ${dirPath}`);
|
|
1448
|
+
process.exit(1);
|
|
1449
|
+
}
|
|
1450
|
+
console.log(`Found ${jsonlFiles.length} JSONL file(s) in directory`);
|
|
1451
|
+
const db = LinesDB.create({ dataDir: dirPath });
|
|
1452
|
+
await db.initialize();
|
|
1453
|
+
const tableNames = db.getTableNames();
|
|
1454
|
+
if (tableNames.length === 0) {
|
|
1455
|
+
console.error(`Error: No tables could be loaded from directory: ${dirPath}`);
|
|
1456
|
+
await db.close();
|
|
1457
|
+
process.exit(1);
|
|
1458
|
+
}
|
|
1459
|
+
console.log(`Loaded ${tableNames.length} table(s): ${tableNames.join(", ")}\n`);
|
|
1460
|
+
try {
|
|
1461
|
+
const transform = runInSandbox(`(${transformStr})`);
|
|
1462
|
+
if (typeof transform !== "function") {
|
|
1463
|
+
console.error("Error: Transform must be a function");
|
|
1464
|
+
await db.close();
|
|
1465
|
+
process.exit(1);
|
|
1466
|
+
}
|
|
1467
|
+
let filter = void 0;
|
|
1468
|
+
if (options.filter) try {
|
|
1469
|
+
filter = JSON.parse(options.filter);
|
|
1470
|
+
} catch {
|
|
1471
|
+
filter = runInSandbox(`(${options.filter})`);
|
|
1472
|
+
}
|
|
1473
|
+
let totalRowsMigrated = 0;
|
|
1474
|
+
let hasErrors = false;
|
|
1475
|
+
for (const tableName of tableNames) try {
|
|
1476
|
+
console.log(`Processing table '${tableName}'...`);
|
|
1477
|
+
const rowsToMigrate = filter ? db.find(tableName, filter) : db.find(tableName);
|
|
1478
|
+
if (rowsToMigrate.length === 0) {
|
|
1479
|
+
console.log(` No rows to migrate`);
|
|
1480
|
+
continue;
|
|
1481
|
+
}
|
|
1482
|
+
console.log(` Found ${rowsToMigrate.length} row(s) to migrate`);
|
|
1483
|
+
const transformedRows = rowsToMigrate.map((row) => transform(row));
|
|
1484
|
+
await db.transaction(async () => {
|
|
1485
|
+
db.batchUpdate(tableName, transformedRows, { validate: true });
|
|
1486
|
+
});
|
|
1487
|
+
console.log(` ✓ ${rowsToMigrate.length} row(s) updated\n`);
|
|
1488
|
+
totalRowsMigrated += rowsToMigrate.length;
|
|
1489
|
+
} catch (error) {
|
|
1490
|
+
hasErrors = true;
|
|
1491
|
+
console.error(styleText("red", ` ✗ Failed to migrate table '${tableName}'`));
|
|
1492
|
+
const formatter = new ErrorFormatter({ verbose: options.verbose });
|
|
1493
|
+
if (error instanceof Error && error.name === "ValidationError") {
|
|
1494
|
+
const validationError = error;
|
|
1495
|
+
if (validationError.validationErrors) {
|
|
1496
|
+
console.error(` Found ${validationError.validationErrors.length} validation error(s):\n`);
|
|
1497
|
+
const rowsToMigrate = filter ? db.find(tableName, filter) : db.find(tableName);
|
|
1498
|
+
const errorInfos = validationError.validationErrors.map(({ rowIndex, rowData, error: rowError }) => ({
|
|
1499
|
+
file: `${dirPath}/${tableName}.jsonl`,
|
|
1500
|
+
rowIndex,
|
|
1501
|
+
issues: rowError.issues,
|
|
1502
|
+
data: rowData,
|
|
1503
|
+
originalData: rowsToMigrate[rowIndex]
|
|
1504
|
+
}));
|
|
1505
|
+
const formatted = formatter.formatValidationErrors(errorInfos);
|
|
1506
|
+
console.error(formatted);
|
|
1507
|
+
}
|
|
1508
|
+
} else if (error instanceof Error) console.error(` ${error.message}`);
|
|
1509
|
+
console.error("");
|
|
1510
|
+
}
|
|
1511
|
+
await db.close();
|
|
1512
|
+
if (hasErrors) {
|
|
1513
|
+
console.error(styleText("red", `\n✗ Migration completed with errors for some tables`));
|
|
1514
|
+
console.log(`Total rows migrated: ${totalRowsMigrated}`);
|
|
1515
|
+
process.exit(1);
|
|
1516
|
+
} else {
|
|
1517
|
+
console.log(styleText("green", `\n✓ Migration completed successfully for all tables`));
|
|
1518
|
+
console.log(`Total rows migrated: ${totalRowsMigrated}`);
|
|
1519
|
+
process.exit(0);
|
|
1520
|
+
}
|
|
1521
|
+
} catch (error) {
|
|
1522
|
+
await db.close();
|
|
1523
|
+
throw error;
|
|
1524
|
+
}
|
|
1525
|
+
}
|
|
1526
|
+
/**
|
|
1527
|
+
* Migrate a single JSONL file
|
|
1528
|
+
*/
|
|
1529
|
+
async function migrateFile(filePath, transformStr, options) {
|
|
1427
1530
|
const tableName = (filePath.split("/").pop() || "").replace(".jsonl", "");
|
|
1428
1531
|
if (!tableName) {
|
|
1429
1532
|
console.error("Error: Invalid file path. Must be a .jsonl file");
|
|
@@ -1520,7 +1623,7 @@ program.command("migrate").description("Migrate data with transformation functio
|
|
|
1520
1623
|
await db.close();
|
|
1521
1624
|
throw error;
|
|
1522
1625
|
}
|
|
1523
|
-
}
|
|
1626
|
+
}
|
|
1524
1627
|
program.parse();
|
|
1525
1628
|
|
|
1526
1629
|
//#endregion
|
package/dist/index.d.cts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.cts","names":[],"sources":["../src/sqlite-adapter.ts","../src/types.ts","../src/database.ts","../src/jsonl-reader.ts","../src/jsonl-writer.ts","../src/schema-loader.ts","../src/directory-scanner.ts","../src/schema.ts","../src/type-generator.ts","../src/validator.ts","../src/jsonl-migration.ts","../src/runtime.ts"],"sourcesContent":[],"mappings":";;;;;;;AASA;AAMA;;UANiB,cAAA;wBACO;ECPZ,IAAA,CAAA,GAAK,EAAA,MAAA,CAAA,EAAG,IAAA;EACR,KAAA,EAAA,EAAA,IAAA;;AACY,UDUP,eAAA,CCVO;EACP,GAAA,CAAA,GAAA,MAAA,EAAA,GAAA,EAAA,CAAA,EAAA;IAAQ,OAAA,EAAA,MAAA;IACJ,eAAA,EAAA,MAAA,GAAA,MAAA;EAAO,CAAA;EAAxB,GAAA,CAAA,GAAA,MAAA,EAAA,GAAA,EAAA,CAAA,EAAA,GAAA;EAAgB,GAAA,CAAA,GAAA,MAAA,EAAA,GAAA,EAAA,CAAA,EAAA,GAAA,EAAA;AACpB;;;KALY,KAAA,GAAQ;KACR,6BACI,QAAQ,sBACP,QAAQ,SACrB,iBAAiB,OAAO;ADEX,KCDL,oBDCmB,CACP,MAAA,CAAA,GCFmB,gBAAA,CAAiB,MDErB,CCF4B,MDE5B,CAAA;AAKtB,KCNL,mBAAA,GAAsB,gBAAA,CAAiB,KDMnB;KCCpB,kBAAgB,YAAU,qCAAqC;KAC/D,mBAAiB,YAAU,qCAAqC;AAdhE,UAgBK,oBAAA,CAhBS;EACd,MAAA,EAAA,MAAA;EACI,UAAA,EAAA;IAAQ,KAAA,EAAA,MAAA;IACP,MAAA,EAAA,MAAA;EAAQ,CAAA;EACJ,QAAA,CAAA,EAAA,SAAA,GAAA,UAAA,GAAA,UAAA,GAAA,WAAA;EAAO,QAAA,CAAA,EAAA,SAAA,GAAA,UAAA,GAAA,UAAA,GAAA,WAAA;;AAAR,UAsBH,eAAA,CAtBG;EACR,IAAA,CAAA,EAAA,MAAA;EACA,OAAA,EAAA,MAAA,EAAA;EAOA,MAAA,CAAA,EAAA,OAAU;;AAAgB,UAmBrB,WAAA,CAnBqB;EAAqC,IAAA,EAAA,MAAA;EAAC,OAAA,EAqBjE,gBArBiE,EAAA;EAChE,WAAA,CAAA,EAqBI,oBArBO,EAAA;EAAM,OAAA,CAAA,EAsBjB,eAtBiB,EAAA;;AAA+C,UAyB3D,gBAAA,CAzB2D;EAAC,IAAA,EAAA,MAAA;EAE5D,IAAA,EAAA,MAAA,GAAA,SAAoB,GAAA,MAAA,GAAA,MAAA,GAAA,MAAA,GAAA,MAAA;EAUpB,UAAA,CAAA,EAAA,OAAe;EAMf,OAAA,CAAA,EAAA,OAAW;EAEjB,MAAA,CAAA,EAAA,OAAA;EACK,SAAA,CAAA,EAAA,SAAA;;AACW,KAYf,SAAA,GAAY,MAZG,CAAA,MAAA,EAYY,KAZZ,CAAA;AAGV,cAUI,YAVY,EAAA,OAAA,MAAA;AASrB,UAGK,cAHsB,CAAA,gBAGS,SAHlB,GAG8B,SAH9B,CAAA,CAAA;EACT,OAAA,EAAA,MAA2B;EAE/B,UAEL,YAAA,EAFmB,EAEH,OAFG;;AAEP,UAaP,WAAA,CAbO;EAaP,SAAA,EAAA,MAAW;EAOX,MAAA,CAAA,EALN,WAKsB;EAET,eAAA,CAAA,EAAA,OAAA;EAAd,gBAAA,CAAA,EALW,cAKX;;AAFoC,UAA7B,eAAA,SAAwB,KAAK,CAAA;EAKlC,IAAA,EAAA,iBAAS;EACJ,MAAA,EAJP,aAIiB,CAJH,mBAKE,CAAA;AAE1B;AAGY,KAPA,SAAA,GAOU,MAAA,GAAM,MAAa,GAAC,OAAA,GAAA,IAAA,GAPiB,UAOjB,GAP8B,SAO9B;AAE9B,UARK,UAAA,CAQM;EAAW,CAAA,GAAA,EAAA,MAAA,CAAA,EAPjB,SAOiB;;AACJ,KANlB,SAAA,GAAY,SAMM,EAAA;AAAE,KAHpB,UAGoB,CAAA,GAAA,CAAA,GAHJ,GAGI,GAAA,CAAA,CAAA,KAAA,EAHS,GAGT,EAAA,GAAA,OAAA,CAAA;AAAb,KADP,WACO,CAAA,YADe,KACf,CAAA,GAAA,QAAU,MAAf,GAAe,IAAV,UAAU,CAAC,GAAD,CAAG,CAAH,CAAA,CAAA,EAG7B;AAAqC,KAAzB,cAAyB,CAAA,YAAA,KAAA,CAAA,GACjC,WADiC,CACrB,GADqB,CAAA,GAEjC,mBAFiC,CAEb,GAFa,CAAA;AACrB,KAGJ,mBAHI,CAAA,YAG0B,KAH1B,CAAA,GAGmC,KAHnC,CAGyC,WAHzC,CAGqD,GAHrD,CAAA,GAG0D,mBAH1D,CAG8E,GAH9E,CAAA,CAAA;;;cC3EH,uBAAuB;EFXnB,QAAA,EAAA;EAMA,QAAA,MAAA;;;;ECZL,QAAK,aAAG;EACR,QAAA,WAAc,CAAA;EACV,OAAA,MAAA,CAAA,eC4Be,SD5Bf,CAAA,CAAA,MAAA,EC6BJ,cD7BI,CC6BW,MD7BX,CAAA,EAAA,MAAA,CAAA,EAAA,MAAA,CAAA,EC+BX,OD/BW,CC+BH,MD/BG,CAAA;EAAQ;;;;EAEI,UAAA,CAAA,CAAA,ECqCN,ODrCM,CAAA,IAAA,CAAA;EAAxB;;AACJ;EACY,QAAA,
|
|
1
|
+
{"version":3,"file":"index.d.cts","names":[],"sources":["../src/sqlite-adapter.ts","../src/types.ts","../src/database.ts","../src/jsonl-reader.ts","../src/jsonl-writer.ts","../src/schema-loader.ts","../src/directory-scanner.ts","../src/schema.ts","../src/type-generator.ts","../src/validator.ts","../src/jsonl-migration.ts","../src/runtime.ts"],"sourcesContent":[],"mappings":";;;;;;;AASA;AAMA;;UANiB,cAAA;wBACO;ECPZ,IAAA,CAAA,GAAK,EAAA,MAAA,CAAA,EAAG,IAAA;EACR,KAAA,EAAA,EAAA,IAAA;;AACY,UDUP,eAAA,CCVO;EACP,GAAA,CAAA,GAAA,MAAA,EAAA,GAAA,EAAA,CAAA,EAAA;IAAQ,OAAA,EAAA,MAAA;IACJ,eAAA,EAAA,MAAA,GAAA,MAAA;EAAO,CAAA;EAAxB,GAAA,CAAA,GAAA,MAAA,EAAA,GAAA,EAAA,CAAA,EAAA,GAAA;EAAgB,GAAA,CAAA,GAAA,MAAA,EAAA,GAAA,EAAA,CAAA,EAAA,GAAA,EAAA;AACpB;;;KALY,KAAA,GAAQ;KACR,6BACI,QAAQ,sBACP,QAAQ,SACrB,iBAAiB,OAAO;ADEX,KCDL,oBDCmB,CACP,MAAA,CAAA,GCFmB,gBAAA,CAAiB,MDErB,CCF4B,MDE5B,CAAA;AAKtB,KCNL,mBAAA,GAAsB,gBAAA,CAAiB,KDMnB;KCCpB,kBAAgB,YAAU,qCAAqC;KAC/D,mBAAiB,YAAU,qCAAqC;AAdhE,UAgBK,oBAAA,CAhBS;EACd,MAAA,EAAA,MAAA;EACI,UAAA,EAAA;IAAQ,KAAA,EAAA,MAAA;IACP,MAAA,EAAA,MAAA;EAAQ,CAAA;EACJ,QAAA,CAAA,EAAA,SAAA,GAAA,UAAA,GAAA,UAAA,GAAA,WAAA;EAAO,QAAA,CAAA,EAAA,SAAA,GAAA,UAAA,GAAA,UAAA,GAAA,WAAA;;AAAR,UAsBH,eAAA,CAtBG;EACR,IAAA,CAAA,EAAA,MAAA;EACA,OAAA,EAAA,MAAA,EAAA;EAOA,MAAA,CAAA,EAAA,OAAU;;AAAgB,UAmBrB,WAAA,CAnBqB;EAAqC,IAAA,EAAA,MAAA;EAAC,OAAA,EAqBjE,gBArBiE,EAAA;EAChE,WAAA,CAAA,EAqBI,oBArBO,EAAA;EAAM,OAAA,CAAA,EAsBjB,eAtBiB,EAAA;;AAA+C,UAyB3D,gBAAA,CAzB2D;EAAC,IAAA,EAAA,MAAA;EAE5D,IAAA,EAAA,MAAA,GAAA,SAAoB,GAAA,MAAA,GAAA,MAAA,GAAA,MAAA,GAAA,MAAA;EAUpB,UAAA,CAAA,EAAA,OAAe;EAMf,OAAA,CAAA,EAAA,OAAW;EAEjB,MAAA,CAAA,EAAA,OAAA;EACK,SAAA,CAAA,EAAA,SAAA;;AACW,KAYf,SAAA,GAAY,MAZG,CAAA,MAAA,EAYY,KAZZ,CAAA;AAGV,cAUI,YAVY,EAAA,OAAA,MAAA;AASrB,UAGK,cAHsB,CAAA,gBAGS,SAHlB,GAG8B,SAH9B,CAAA,CAAA;EACT,OAAA,EAAA,MAA2B;EAE/B,UAEL,YAAA,EAFmB,EAEH,OAFG;;AAEP,UAaP,WAAA,CAbO;EAaP,SAAA,EAAA,MAAW;EAOX,MAAA,CAAA,EALN,WAKsB;EAET,eAAA,CAAA,EAAA,OAAA;EAAd,gBAAA,CAAA,EALW,cAKX;;AAFoC,UAA7B,eAAA,SAAwB,KAAK,CAAA;EAKlC,IAAA,EAAA,iBAAS;EACJ,MAAA,EAJP,aAIiB,CAJH,mBAKE,CAAA;AAE1B;AAGY,KAPA,SAAA,GAOU,MAAA,GAAM,MAAa,GAAC,OAAA,GAAA,IAAA,GAPiB,UAOjB,GAP8B,SAO9B;AAE9B,UARK,UAAA,CAQM;EAAW,CAAA,GAAA,EAAA,MAAA,CAAA,EAPjB,SAOiB;;AACJ,KANlB,SAAA,GAAY,SAMM,EAAA;AAAE,KAHpB,UAGoB,CAAA,GAAA,CAAA,GAHJ,GAGI,GAAA,CAAA,CAAA,KAAA,EAHS,GAGT,EAAA,GAAA,OAAA,CAAA;AAAb,KADP,WACO,CAAA,YADe,KACf,CAAA,GAAA,QAAU,MAAf,GAAe,IAAV,UAAU,CAAC,GAAD,CAAG,CAAH,CAAA,CAAA,EAG7B;AAAqC,KAAzB,cAAyB,CAAA,YAAA,KAAA,CAAA,GACjC,WADiC,CACrB,GADqB,CAAA,GAEjC,mBAFiC,CAEb,GAFa,CAAA;AACrB,KAGJ,mBAHI,CAAA,YAG0B,KAH1B,CAAA,GAGmC,KAHnC,CAGyC,WAHzC,CAGqD,GAHrD,CAAA,GAG0D,mBAH1D,CAG8E,GAH9E,CAAA,CAAA;;;cC3EH,uBAAuB;EFXnB,QAAA,EAAA;EAMA,QAAA,MAAA;;;;ECZL,QAAK,aAAG;EACR,QAAA,WAAc,CAAA;EACV,OAAA,MAAA,CAAA,eC4Be,SD5Bf,CAAA,CAAA,MAAA,EC6BJ,cD7BI,CC6BW,MD7BX,CAAA,EAAA,MAAA,CAAA,EAAA,MAAA,CAAA,EC+BX,OD/BW,CC+BH,MD/BG,CAAA;EAAQ;;;;EAEI,UAAA,CAAA,CAAA,ECqCN,ODrCM,CAAA,IAAA,CAAA;EAAxB;;AACJ;EACY,QAAA,yBAAmB;EAOnB;;;;EAAgE,QAAA,SAAA;EAChE;;;EAAgE,QAAA,WAAA;EAAC;AAE7E;AAUA;EAMiB,QAAA,cAAW;EAEjB;;;EAEgB,QAAA,eAAA;EAGV;AASjB;AACA;EAEiB,QAAA,UAAc;EAAiB;;;EAEpC,KAAA,CAAA,MAAA,OAAA,CAAA,CAAA,GAAA,EAAA,MAAA,EAAA,MAAA,CAAA,EAAA,CAAA,MAAA,GAAA,MAAA,GAAA,MAAA,GAAA,IAAA,GCmVmC,UDnVnC,CAAA,EAAA,CAAA,ECoVP,GDpVO,EAAA;EAAY;AAaxB;AAOA;EAEwB,QAAA,CAAA,MAAA,OAAA,CAAA,CAAA,GAAA,EAAA,MAAA,EAAA,MAAA,CAAA,EAAA,CAAA,MAAA,GAAA,MAAA,GAAA,MAAA,GAAA,IAAA,GCwUuB,UDxUvB,CAAA,EAAA,CAAA,ECyUnB,GDzUmB,GAAA,IAAA;EAAd;;;EAGE,OAAA,CAAA,GAAA,EAAS,MAAA,EAAA,MAA4D,CAAtB,EAAA,CAAA,MAAa,GAAA,MAAS,GAAA,MAAA,GAAA,IAAA,GCiVlC,UDjVkC,CAAA,EAAA,CAAA,EAAA;IAChE,OAAA,EAAU,MAAA,GAAA,MACV;IAEL,eAAS,EAAA,MAAG,GAAA,MAAS;EAGrB,CAAA;EAEA;;;;EACoB,IAAA,CAAA,YAAA,MCiVT,MDjVS,GAAA,MAAA,CAAA,CAAA,SAAA,ECiVmB,GDjVnB,EAAA,KAAA,CAAA,ECiV8B,cDjV9B,CCiV6C,MDjV7C,CCiVoD,GDjVpD,CAAA,CAAA,CAAA,ECiVuD,MDjVvD,CCiVuD,GDjVvD,CAAA,EAAA;EAAb;;AAGnB;EAAqC,OAAA,CAAA,YAAA,MCyXX,MDzXW,GAAA,MAAA,CAAA,CAAA,SAAA,ECyXiB,GDzXjB,EAAA,KAAA,ECyX2B,cDzX3B,CCyX0C,MDzX1C,CCyXiD,GDzXjD,CAAA,CAAA,CAAA,ECyXoD,MDzXpD,CCyXoD,GDzXpD,CAAA,GAAA,IAAA;EACrB;;;EACZ,QAAA,cAAA;EAAmB;AAEvB;;;EAAyD,QAAA,oBAAA;EAAqC;;;;;;;AC9E9F;EAAoC,MAAA,CAAA,YAAA,MAqkBX,MArkBW,GAAA,MAAA,CAAA,CAAA,SAAA,EAskBrB,GAtkBqB,EAAA,IAAA,EAukB1B,MAvkB0B,CAukBnB,GAvkBmB,CAAA,CAAA,EAAA;IAaL,OAAA,EAAA,MAAA,GAAA,MAAA;IACJ,eAAA,EAAA,MAAA,GAAA,MAAA;EAAf,CAAA;EAEC;;;EAuWkC,WAAA,CAAA,YAAA,MA+OjB,MA/OiB,GAAA,MAAA,CAAA,CAAA,SAAA,EAgPhC,GAhPgC,EAAA,OAAA,EAiPlC,MAjPkC,CAiP3B,GAjP2B,CAAA,EAAA,CAAA,EAAA;IAC1C,OAAA,EAAA,MAAA,GAAA,MAAA;IAU0C,eAAA,EAAA,MAAA,GAAA,MAAA;EAC1C,CAAA;EAW0C;;;;;;EAUwC,MAAA,CAAA,YAAA,MA+P9D,MA/P8D,GAAA,MAAA,CAAA,CAAA,SAAA,EAgQxE,GAhQwE,EAAA,IAAA,EAiQ7E,OAjQ6E,CAiQrE,MAjQqE,CAiQ9D,GAjQ8D,CAAA,CAAA,EAAA,KAAA,EAkQ5E,cAlQ4E,CAkQ7D,MAlQ6D,CAkQtD,GAlQsD,CAAA,CAAA,EAAA,OA2C7D,CA3C6D,EAAA;IAAA,QAAA,CAAA,EAAA,OAAA;EA2C7D,CAAA,CAAA,EAAA;IAA4B,OAAA,EAAA,MAAA,GAAA,MAAA;IAAyB,eAAA,EAAA,MAAA,GAAA,MAAA;EAAO,CAAA;EAAtB;;;;;EAoItD,WAAA,CAAA,YAAA,MAwIoB,MAxIpB,GAAA,MAAA,CAAA,CAAA,SAAA,EAyIK,GAzIL,EAAA,OAAA,EA0IG,KA1IH,CA0IS,OA1IT,CA0IiB,MA1IjB,CA0IwB,GA1IxB,CAAA,CAAA,GA0I8B,MA1I9B,CAAA,MAAA,EAAA,OAAA,CAAA,CAAA,EAAA,OA+BoB,CA/BpB,EAAA;IAAO,QAAA,CAAA,EAAA,OAAA;EA+Ba,CAAA,CAAA,EAAA;IACf,OAAA,EAAA,MAAA,GAAA,MAAA;IACF,eAAA,EAAA,MAAA,GAAA,MAAA;EAAO,CAAA;EA+CK;;;;EAEf,MAAA,CAAA,YAAA,MA0Le,MA1Lf,GAAA,MAAA,CAAA,CAAA,SAAA,EA2LK,GA3LL,EAAA,KAAA,EA4LC,cA5LD,CA4LgB,MA5LhB,CA4LuB,GA5LvB,CAAA,CAAA,CAAA,EAAA;IACgB,OAAA,EAAA,MAAA,GAAA,MAAA;IAAO,eAAA,EAAA,MAAA,GAAA,MAAA;EAAtB,CAAA;EAqDmB;;;EAEI,WAAA,CAAA,YAAA,MAiKJ,MAjKI,GAAA,MAAA,CAAA,CAAA,SAAA,EAkKnB,GAlKmB,EAAA,OAAA,EAmKrB,KAnKqB,CAmKf,OAnKe,CAmKP,MAnKO,CAmKA,GAnKA,CAAA,CAAA,GAmKM,MAnKN,CAAA,MAAA,EAAA,OAAA,CAAA,CAAA,CAAA,EAAA;IAAf,OAAA,EAAA,MAAA,GAAA,MAAA;IAAqB,eAAA,EAAA,MAAA,GAAA,MAAA;EAA3B,CAAA;EAkIY;;;EAEQ,QAAA,cAAA;EAAtB;;;EA+BgB,QAAA,gBAAA;EAAO;;;EAArB,QAAA,2BAAA;EAuKmB;;;EAsDA,QAAA,kBAAA;EAA4B;;;EAAiB,QAAA,oBAAA;EAAR;;;EAwC5C,SAAA,CAAA,SAAA,EAAA,MAAA,CAAA,EA9FO,WA8FP,GAAA,SAAA;;;;ECzoCZ,aAAA,CAAA,CAAW,EAAA,MAAA,EAAA;EAQG;;;;EAEd,QAAA,SAAA;EAAR;;;;EA2CwD,IAAA,CAAA,CAAA,EDkiC7C,OCliC6C,CAAA,IAAA,CAAA;EAAW;;;;ECtD3D,WAAA,CAAA,GAAA,CAAW,CAAA,EAAA,EAAA,CAAA,EAAA,EFkmCQ,OElmCR,CFkmCgB,MElmChB,CAAA,EAAA,GFkmC4B,OElmC5B,CFkmCoC,GElmCpC,CAAA,GFkmCyC,GElmCzC,CAAA,EFkmC6C,OElmC7C,CFkmCqD,GElmCrD,CAAA;EAIqB;;;EAQgB,KAAA,CAAA,CAAA,EFmnC5C,OEnnC4C,CAAA,IAAA,CAAA;EAAO;;;WF8nCzD;AGxoCX;;;cFDa,WAAA;;EHKI;AAMjB;;;uCGHe,YAAY,yBACb,QAAQ,OACjB,QAAQ;EFXD;AACZ;;EACwB,OAAA,IAAA,CAAA,QAAA,EAAA,MAAA,CAAA,EE4Be,OF5Bf,CE4BuB,UF5BvB,EAAA,CAAA;EACP;;;EACW,OAAA,WAAA,CAAA,SAAA,EAAA,MAAA,EAAA,IAAA,EEkDkB,UFlDlB,EAAA,CAAA,EEkDiC,WFlDjC;EAAxB,eAAA,SAAA;;;;cGJS,WAAA;;AJMb;AAMA;uCIR6C,eAAe;;;AHJ5D;EACY,OAAA,MAAA,CAAA,QAAc,EAAA,MAAA,EAAA,IAAA,EGWoB,UHXpB,EAAA,CAAA,EGWmC,OHXnC,CAAA,IAAA,CAAA;;;;cICb,YAAA;;ALIb;AAMA;uCKN6C;;;AJN7C;AACA;EACgB,OAAA,UAAA,CAAA,SAAA,EAAA,MAAA,CAAA,EIqB8B,OJrB9B,CIqBsC,cJrBtC,CAAA;EAAQ;;;EAEH,eAAA,gBAAA;;;;cKHR,gBAAA;;ANKb;AAMA;yCMP+C,QAAQ,YAAY;;;;;;ANCnE;AAMiB,UOVA,aAAA,CPUe;;;;ECZpB,UAAK,CAAA,EAAA,MAAG;EACR;;;EAEK,WAAA,CAAA,EMQD,oBNRC,EAAA;EAAQ;;;EACrB,OAAA,CAAA,EMYQ,eNZR,EAAA;EAAgB;AACpB;AACA;AAOA;EAA4B,QAAA,CAAA,EAAA,CAAA,MAAA,EMSN,KNTM,EAAA,GMSI,KNTJ;;;;AAC5B;;AAAuC,UMetB,mBNfsB,CAAA,cMeY,KNfZ,GMeoB,KNfpB,EAAA,eMe0C,KNf1C,GMekD,KNflD,CAAA,SMgB7B,cNhB6B,CMgBd,KNhBc,EMgBP,MNhBO,CAAA,CAAA;EAAqC;;AAE5E;AAUA;EAMiB,QAAA,CAAA,EAAA,CAAA,MAAW,EMGN,MNHM,EAAA,GMGK,KNHL;EAEjB;;;EAEgB,UAAA,CAAA,EAAA,MAAA;EAGV;AASjB;AACA;EAEiB,WAAA,CAAA,EMND,oBNMe,EAAA;EAAiB;;;EAEpC,OAAA,CAAA,EMHA,eNGA,EAAA;;AAaZ;AAOA;;;;;AAKA;AACA;AAGA;AAGA;AAEA;;;;;;;AAIA;;;;;;;AAIA;;;;;;;;iBMVgB,2BAA2B,sBAAsB,eACvD,eAAe,OAAO,6BACV,0BAA0B,WAAW,SACxD,oBAAoB,OAAO;;;ALvE9B;AAAoC,iBK8GpB,WL9GoB,CAAA,cK8GM,KL9GN,EAAA,eK8G4B,KL9G5B,CAAA,CAAA,MAAA,EK+G1B,cL/G0B,CK+GX,KL/GW,EK+GJ,ML/GI,CAAA,CAAA,EAAA,MAAA,IKgHvB,mBLhHuB,CKgHH,KLhHG,EKgHI,MLhHJ,CAAA;;;UMhBnB,oBAAA;;;ARKjB;AAMiB,cQDJ,aAAA,CRCmB;;;;ECZpB,QAAK,WAAG;EACR,WAAA,CAAA,OAAc,EOgBH,oBPhBG;EACV;;;EACS,QAAA,CAAA,CAAA,EO6BL,OP7BK,CAAA,MAAA,CAAA;EACJ;;;EAAD,QAAA,UAAA;EACR;AACZ;AAOA;EAA4B,QAAA,wBAAA;EAAU,QAAA,sBAAA;EAAqC,QAAA,cAAA;;;;UQT1D,gBAAA;;ETEA,MAAA,ESAP,qBTCc,EAAA;EAKP,QAAA,EAAA,MAAA,EAAe;;USFf,qBAAA;;ERVL,SAAK,EAAA,MAAA;EACL,QAAA,EAAA,MAAA;EACI,MAAA,EQYN,aRZM,CQYQ,mBRZR,CAAA;EAAQ,IAAA,CAAA,EAAA,QAAA,GAAA,YAAA;EACP,eAAA,CAAA,EAAA;IAAQ,MAAA,EAAA,MAAA;IACJ,KAAA,EAAA,OAAA;IAAO,eAAA,EAAA,MAAA;IAAxB,gBAAA,EAAA,MAAA;EAAgB,CAAA;AACpB;AACY,UQkBK,gBAAA,CRlBc;EAOnB,IAAA,EAAA,MAAA;EAAgB,WAAA,CAAA,EAAA,MAAA;;AAA+C,cQgB9D,SAAA,CRhB8D;EAAC,QAAA,IAAA;EAChE,QAAA,WAAW;EAAM,WAAA,CAAA,OAAA,EQmBN,gBRnBM;EAAU;;;EAEtB,QAAA,CAAA,CAAA,EQyBG,ORzBH,CQyBW,gBRzBS,CAAA;EAUpB;AAMjB;;EAGgB,QAAA,iBAAA;EACJ;;AAGZ;AASA;EACqB,QAAA,oBAA2B;EAE/B;;;EAEW,QAAA,iBAAA;EAAhB;;AAaZ;EAOiB,QAAA,eAAgB;EAET;;;EAFsB,QAAA,eAAA;EAKlC;AACZ;AAGA;EAGY,QAAA,sBAAgB;EAEhB;;;EACkB,QAAA,eAAA;EAAE;;;EAGpB,QAAA,YAAc;;;;USzFT,sBAAA;;EVIA,SAAA,EAAA,MAAc;EAMd,IAAA,EUPT,UVOS,EAAe;;;;ACZhC;AACA;AACgB,iBSUM,oBAAA,CTVN,OAAA,ESUoC,sBTVpC,CAAA,ESU6D,OTV7D,CAAA,IAAA,CAAA;;;;;;ADIC,KWLL,kBAAA,GXMY,MAAA,GAAA,SAAe;AAKtB,iBWTD,aAAA,CAAA,CXSgB,EWTC,kBXSD;cWAnB,SAAO"}
|
package/package.json
CHANGED
package/src/cli.ts
CHANGED
|
@@ -11,7 +11,7 @@ import { ErrorFormatter } from './error-formatter.js';
|
|
|
11
11
|
import type { ValidationError } from './types.js';
|
|
12
12
|
import { Command } from 'commander';
|
|
13
13
|
import { styleText } from 'node:util';
|
|
14
|
-
import { writeFile } from 'node:fs/promises';
|
|
14
|
+
import { writeFile, stat, readdir } from 'node:fs/promises';
|
|
15
15
|
import { runInNewContext } from 'node:vm';
|
|
16
16
|
|
|
17
17
|
const originalEmitWarning = process.emitWarning;
|
|
@@ -161,191 +161,375 @@ program
|
|
|
161
161
|
program
|
|
162
162
|
.command('migrate')
|
|
163
163
|
.description('Migrate data with transformation function')
|
|
164
|
-
.argument('<
|
|
164
|
+
.argument('<path>', 'File or directory path to migrate')
|
|
165
165
|
.argument('<transform>', 'Transform function (e.g., "(row) => ({ ...row, age: row.age + 1 })")')
|
|
166
166
|
.option('-f, --filter <expr>', 'Filter expression')
|
|
167
167
|
.option('-e, --errorOutput <path>', 'Output file path for transformed data when migration fails')
|
|
168
168
|
.option('-v, --verbose', 'Show verbose error output', false)
|
|
169
169
|
.action(
|
|
170
170
|
async (
|
|
171
|
-
|
|
171
|
+
path: string,
|
|
172
172
|
transformStr: string,
|
|
173
173
|
options: { filter?: string; errorOutput?: string; verbose: boolean },
|
|
174
174
|
) => {
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
175
|
+
try {
|
|
176
|
+
const stats = await stat(path);
|
|
177
|
+
|
|
178
|
+
if (stats.isDirectory()) {
|
|
179
|
+
// Migrate all JSONL files in directory
|
|
180
|
+
await migrateDirectory(path, transformStr, options);
|
|
181
|
+
} else if (stats.isFile() && path.endsWith('.jsonl')) {
|
|
182
|
+
// Migrate single file
|
|
183
|
+
await migrateFile(path, transformStr, options);
|
|
184
|
+
} else {
|
|
185
|
+
console.error(`Error: Invalid path: ${path}. Must be a directory or .jsonl file.`);
|
|
186
|
+
process.exit(1);
|
|
187
|
+
}
|
|
188
|
+
} catch (error) {
|
|
189
|
+
if (
|
|
190
|
+
error instanceof Error &&
|
|
191
|
+
'code' in error &&
|
|
192
|
+
(error as NodeJS.ErrnoException).code === 'ENOENT'
|
|
193
|
+
) {
|
|
194
|
+
console.error(`Error: Path not found: ${path}`);
|
|
195
|
+
} else {
|
|
196
|
+
console.error(`Error: ${String(error)}`);
|
|
197
|
+
}
|
|
181
198
|
process.exit(1);
|
|
182
199
|
}
|
|
200
|
+
},
|
|
201
|
+
);
|
|
183
202
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
203
|
+
/**
|
|
204
|
+
* Migrate all JSONL files in a directory
|
|
205
|
+
*/
|
|
206
|
+
async function migrateDirectory(
|
|
207
|
+
dirPath: string,
|
|
208
|
+
transformStr: string,
|
|
209
|
+
options: { filter?: string; errorOutput?: string; verbose: boolean },
|
|
210
|
+
) {
|
|
211
|
+
// Find all JSONL files in directory
|
|
212
|
+
const entries = await readdir(dirPath, { withFileTypes: true });
|
|
213
|
+
const jsonlFiles = entries
|
|
214
|
+
.filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl'))
|
|
215
|
+
.map((entry) => entry.name);
|
|
216
|
+
|
|
217
|
+
if (jsonlFiles.length === 0) {
|
|
218
|
+
console.error(`Error: No JSONL files found in directory: ${dirPath}`);
|
|
219
|
+
process.exit(1);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
console.log(`Found ${jsonlFiles.length} JSONL file(s) in directory`);
|
|
223
|
+
|
|
224
|
+
// Initialize database
|
|
225
|
+
const db = LinesDB.create({ dataDir: dirPath });
|
|
226
|
+
await db.initialize();
|
|
227
|
+
|
|
228
|
+
const tableNames = db.getTableNames();
|
|
229
|
+
if (tableNames.length === 0) {
|
|
230
|
+
console.error(`Error: No tables could be loaded from directory: ${dirPath}`);
|
|
231
|
+
await db.close();
|
|
232
|
+
process.exit(1);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
console.log(`Loaded ${tableNames.length} table(s): ${tableNames.join(', ')}\n`);
|
|
236
|
+
|
|
237
|
+
try {
|
|
238
|
+
// Parse transform function
|
|
239
|
+
const transform = runInSandbox<unknown>(`(${transformStr})`);
|
|
240
|
+
|
|
241
|
+
if (typeof transform !== 'function') {
|
|
242
|
+
console.error('Error: Transform must be a function');
|
|
243
|
+
await db.close();
|
|
244
|
+
process.exit(1);
|
|
245
|
+
}
|
|
191
246
|
|
|
247
|
+
// Parse filter if provided
|
|
248
|
+
let filter: unknown = undefined;
|
|
249
|
+
if (options.filter) {
|
|
192
250
|
try {
|
|
193
|
-
|
|
194
|
-
|
|
251
|
+
filter = JSON.parse(options.filter);
|
|
252
|
+
} catch {
|
|
253
|
+
filter = runInSandbox(`(${options.filter})`);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
195
256
|
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
process.exit(1);
|
|
199
|
-
}
|
|
257
|
+
let totalRowsMigrated = 0;
|
|
258
|
+
let hasErrors = false;
|
|
200
259
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
// Try JSON parse first
|
|
206
|
-
filter = JSON.parse(options.filter);
|
|
207
|
-
} catch {
|
|
208
|
-
// Fall back to eval for JavaScript expressions
|
|
209
|
-
filter = runInSandbox(`(${options.filter})`);
|
|
210
|
-
}
|
|
211
|
-
}
|
|
260
|
+
// Process each table
|
|
261
|
+
for (const tableName of tableNames) {
|
|
262
|
+
try {
|
|
263
|
+
console.log(`Processing table '${tableName}'...`);
|
|
212
264
|
|
|
213
265
|
// Get rows to migrate
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
? db.find(tableName, filter as Parameters<typeof db.find>[1])
|
|
218
|
-
: db.find(tableName);
|
|
219
|
-
} catch (error) {
|
|
220
|
-
console.error(`Error: Failed to access table '${tableName}'`);
|
|
221
|
-
console.error(` ${error instanceof Error ? error.message : String(error)}`);
|
|
222
|
-
console.error(`\nThe table may have failed to load during initialization.`);
|
|
223
|
-
console.error(`Check the table's data and schema for any constraint violations.`);
|
|
224
|
-
await db.close();
|
|
225
|
-
process.exit(1);
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
console.log(`Found ${rowsToMigrate.length} row(s) to migrate in table '${tableName}'`);
|
|
266
|
+
const rowsToMigrate = filter
|
|
267
|
+
? db.find(tableName, filter as Parameters<typeof db.find>[1])
|
|
268
|
+
: db.find(tableName);
|
|
229
269
|
|
|
230
270
|
if (rowsToMigrate.length === 0) {
|
|
231
|
-
console.log(
|
|
232
|
-
|
|
233
|
-
process.exit(0);
|
|
271
|
+
console.log(` No rows to migrate`);
|
|
272
|
+
continue;
|
|
234
273
|
}
|
|
235
274
|
|
|
275
|
+
console.log(` Found ${rowsToMigrate.length} row(s) to migrate`);
|
|
276
|
+
|
|
236
277
|
// Apply transformation
|
|
237
278
|
const transformedRows = rowsToMigrate.map((row) => transform(row));
|
|
238
279
|
|
|
239
280
|
// Perform the migration in a transaction
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
validate: true,
|
|
244
|
-
});
|
|
281
|
+
await db.transaction(async () => {
|
|
282
|
+
db.batchUpdate(tableName, transformedRows as Parameters<typeof db.batchUpdate>[1], {
|
|
283
|
+
validate: true,
|
|
245
284
|
});
|
|
285
|
+
});
|
|
246
286
|
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
} catch (error) {
|
|
253
|
-
await db.close();
|
|
254
|
-
|
|
255
|
-
// Write transformed data to error output file if --errorOutput is specified
|
|
256
|
-
if (options.errorOutput) {
|
|
257
|
-
try {
|
|
258
|
-
const jsonlContent = transformedRows.map((row) => JSON.stringify(row)).join('\n');
|
|
259
|
-
await writeFile(options.errorOutput, jsonlContent, 'utf-8');
|
|
260
|
-
console.error(
|
|
261
|
-
styleText(
|
|
262
|
-
'yellow',
|
|
263
|
-
`\n⚠ Transformed data (${transformedRows.length} rows) written to: ${options.errorOutput}`,
|
|
264
|
-
),
|
|
265
|
-
);
|
|
266
|
-
} catch (writeError) {
|
|
267
|
-
console.error(
|
|
268
|
-
styleText(
|
|
269
|
-
'red',
|
|
270
|
-
`\n✗ Failed to write error output file: ${writeError instanceof Error ? writeError.message : String(writeError)}`,
|
|
271
|
-
),
|
|
272
|
-
);
|
|
273
|
-
}
|
|
274
|
-
}
|
|
287
|
+
console.log(` ✓ ${rowsToMigrate.length} row(s) updated\n`);
|
|
288
|
+
totalRowsMigrated += rowsToMigrate.length;
|
|
289
|
+
} catch (error) {
|
|
290
|
+
hasErrors = true;
|
|
291
|
+
console.error(styleText('red', ` ✗ Failed to migrate table '${tableName}'`));
|
|
275
292
|
|
|
276
|
-
|
|
277
|
-
console.error(formatter.formatMigrationFailureHeader());
|
|
278
|
-
|
|
279
|
-
// Display detailed error information
|
|
280
|
-
if (error instanceof Error && error.name === 'ValidationError') {
|
|
281
|
-
const validationError = error as ValidationError & {
|
|
282
|
-
validationErrors?: Array<{
|
|
283
|
-
rowIndex: number;
|
|
284
|
-
rowData: unknown;
|
|
285
|
-
pkValue: unknown;
|
|
286
|
-
error: ValidationError;
|
|
287
|
-
}>;
|
|
288
|
-
};
|
|
289
|
-
|
|
290
|
-
// Display all validation errors
|
|
291
|
-
if (validationError.validationErrors) {
|
|
292
|
-
console.error(
|
|
293
|
-
`\nFound ${validationError.validationErrors.length} validation error(s) in transformed data:\n`,
|
|
294
|
-
);
|
|
295
|
-
|
|
296
|
-
const errorInfos = validationError.validationErrors.map(
|
|
297
|
-
({ rowIndex, rowData, error: rowError }) => ({
|
|
298
|
-
file: filePath,
|
|
299
|
-
rowIndex,
|
|
300
|
-
issues: rowError.issues,
|
|
301
|
-
data: rowData,
|
|
302
|
-
originalData: rowsToMigrate[rowIndex],
|
|
303
|
-
}),
|
|
304
|
-
);
|
|
305
|
-
|
|
306
|
-
const formatted = formatter.formatValidationErrors(errorInfos);
|
|
307
|
-
console.error(formatted);
|
|
308
|
-
} else {
|
|
309
|
-
// Fallback for single validation error (backward compatibility)
|
|
310
|
-
console.error('\nValidation error:\n');
|
|
311
|
-
const errorInfo = {
|
|
312
|
-
file: filePath,
|
|
313
|
-
rowIndex: 0,
|
|
314
|
-
issues: validationError.issues,
|
|
315
|
-
};
|
|
316
|
-
const formatted = formatter.formatValidationErrors([errorInfo]);
|
|
317
|
-
console.error(formatted);
|
|
318
|
-
}
|
|
319
|
-
} else if (error instanceof Error) {
|
|
320
|
-
console.error(`\n ${error.message}`);
|
|
293
|
+
const formatter = new ErrorFormatter({ verbose: options.verbose });
|
|
321
294
|
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
295
|
+
if (error instanceof Error && error.name === 'ValidationError') {
|
|
296
|
+
const validationError = error as ValidationError & {
|
|
297
|
+
validationErrors?: Array<{
|
|
298
|
+
rowIndex: number;
|
|
299
|
+
rowData: unknown;
|
|
300
|
+
pkValue: unknown;
|
|
301
|
+
error: ValidationError;
|
|
302
|
+
}>;
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
if (validationError.validationErrors) {
|
|
306
|
+
console.error(
|
|
307
|
+
` Found ${validationError.validationErrors.length} validation error(s):\n`,
|
|
308
|
+
);
|
|
326
309
|
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
310
|
+
const rowsToMigrate = filter
|
|
311
|
+
? db.find(tableName, filter as Parameters<typeof db.find>[1])
|
|
312
|
+
: db.find(tableName);
|
|
313
|
+
|
|
314
|
+
const errorInfos = validationError.validationErrors.map(
|
|
315
|
+
({ rowIndex, rowData, error: rowError }) => ({
|
|
316
|
+
file: `${dirPath}/${tableName}.jsonl`,
|
|
317
|
+
rowIndex,
|
|
318
|
+
issues: rowError.issues,
|
|
319
|
+
data: rowData,
|
|
320
|
+
originalData: rowsToMigrate[rowIndex],
|
|
321
|
+
}),
|
|
322
|
+
);
|
|
323
|
+
|
|
324
|
+
const formatted = formatter.formatValidationErrors(errorInfos);
|
|
325
|
+
console.error(formatted);
|
|
339
326
|
}
|
|
327
|
+
} else if (error instanceof Error) {
|
|
328
|
+
console.error(` ${error.message}`);
|
|
329
|
+
}
|
|
340
330
|
|
|
341
|
-
|
|
342
|
-
|
|
331
|
+
console.error('');
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
await db.close();
|
|
336
|
+
|
|
337
|
+
if (hasErrors) {
|
|
338
|
+
console.error(styleText('red', `\n✗ Migration completed with errors for some tables`));
|
|
339
|
+
console.log(`Total rows migrated: ${totalRowsMigrated}`);
|
|
340
|
+
process.exit(1);
|
|
341
|
+
} else {
|
|
342
|
+
console.log(styleText('green', `\n✓ Migration completed successfully for all tables`));
|
|
343
|
+
console.log(`Total rows migrated: ${totalRowsMigrated}`);
|
|
344
|
+
process.exit(0);
|
|
345
|
+
}
|
|
346
|
+
} catch (error) {
|
|
347
|
+
await db.close();
|
|
348
|
+
throw error;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Migrate a single JSONL file
|
|
354
|
+
*/
|
|
355
|
+
async function migrateFile(
|
|
356
|
+
filePath: string,
|
|
357
|
+
transformStr: string,
|
|
358
|
+
options: { filter?: string; errorOutput?: string; verbose: boolean },
|
|
359
|
+
) {
|
|
360
|
+
// Extract table name from file path
|
|
361
|
+
const fileName = filePath.split('/').pop() || '';
|
|
362
|
+
const tableName = fileName.replace('.jsonl', '');
|
|
363
|
+
|
|
364
|
+
if (!tableName) {
|
|
365
|
+
console.error('Error: Invalid file path. Must be a .jsonl file');
|
|
366
|
+
process.exit(1);
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
// Get directory from file path
|
|
370
|
+
const lastSlashIndex = filePath.lastIndexOf('/');
|
|
371
|
+
const dataDir = lastSlashIndex > 0 ? filePath.substring(0, lastSlashIndex) : '.';
|
|
372
|
+
|
|
373
|
+
// Initialize database
|
|
374
|
+
const db = LinesDB.create({ dataDir });
|
|
375
|
+
await db.initialize();
|
|
376
|
+
|
|
377
|
+
try {
|
|
378
|
+
// Parse transform function
|
|
379
|
+
const transform = runInSandbox<unknown>(`(${transformStr})`);
|
|
380
|
+
|
|
381
|
+
if (typeof transform !== 'function') {
|
|
382
|
+
console.error('Error: Transform must be a function');
|
|
383
|
+
process.exit(1);
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
// Parse filter if provided
|
|
387
|
+
let filter: unknown = undefined;
|
|
388
|
+
if (options.filter) {
|
|
389
|
+
try {
|
|
390
|
+
// Try JSON parse first
|
|
391
|
+
filter = JSON.parse(options.filter);
|
|
392
|
+
} catch {
|
|
393
|
+
// Fall back to eval for JavaScript expressions
|
|
394
|
+
filter = runInSandbox(`(${options.filter})`);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
// Get rows to migrate
|
|
399
|
+
let rowsToMigrate;
|
|
400
|
+
try {
|
|
401
|
+
rowsToMigrate = filter
|
|
402
|
+
? db.find(tableName, filter as Parameters<typeof db.find>[1])
|
|
403
|
+
: db.find(tableName);
|
|
404
|
+
} catch (error) {
|
|
405
|
+
console.error(`Error: Failed to access table '${tableName}'`);
|
|
406
|
+
console.error(` ${error instanceof Error ? error.message : String(error)}`);
|
|
407
|
+
console.error(`\nThe table may have failed to load during initialization.`);
|
|
408
|
+
console.error(`Check the table's data and schema for any constraint violations.`);
|
|
409
|
+
await db.close();
|
|
410
|
+
process.exit(1);
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
console.log(`Found ${rowsToMigrate.length} row(s) to migrate in table '${tableName}'`);
|
|
414
|
+
|
|
415
|
+
if (rowsToMigrate.length === 0) {
|
|
416
|
+
console.log('No rows to migrate. Exiting.');
|
|
417
|
+
await db.close();
|
|
418
|
+
process.exit(0);
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// Apply transformation
|
|
422
|
+
const transformedRows = rowsToMigrate.map((row) => transform(row));
|
|
423
|
+
|
|
424
|
+
// Perform the migration in a transaction
|
|
425
|
+
try {
|
|
426
|
+
await db.transaction(async () => {
|
|
427
|
+
db.batchUpdate(tableName, transformedRows as Parameters<typeof db.batchUpdate>[1], {
|
|
428
|
+
validate: true,
|
|
429
|
+
});
|
|
430
|
+
});
|
|
431
|
+
|
|
432
|
+
await db.close();
|
|
433
|
+
|
|
434
|
+
console.log(`\nMigration completed successfully:`);
|
|
435
|
+
console.log(` ✓ ${rowsToMigrate.length} row(s) updated`);
|
|
436
|
+
process.exit(0);
|
|
437
|
+
} catch (error) {
|
|
438
|
+
await db.close();
|
|
439
|
+
|
|
440
|
+
// Write transformed data to error output file if --errorOutput is specified
|
|
441
|
+
if (options.errorOutput) {
|
|
442
|
+
try {
|
|
443
|
+
const jsonlContent = transformedRows.map((row) => JSON.stringify(row)).join('\n');
|
|
444
|
+
await writeFile(options.errorOutput, jsonlContent, 'utf-8');
|
|
445
|
+
console.error(
|
|
446
|
+
styleText(
|
|
447
|
+
'yellow',
|
|
448
|
+
`\n⚠ Transformed data (${transformedRows.length} rows) written to: ${options.errorOutput}`,
|
|
449
|
+
),
|
|
450
|
+
);
|
|
451
|
+
} catch (writeError) {
|
|
452
|
+
console.error(
|
|
453
|
+
styleText(
|
|
454
|
+
'red',
|
|
455
|
+
`\n✗ Failed to write error output file: ${writeError instanceof Error ? writeError.message : String(writeError)}`,
|
|
456
|
+
),
|
|
457
|
+
);
|
|
343
458
|
}
|
|
344
|
-
} catch (error) {
|
|
345
|
-
await db.close();
|
|
346
|
-
throw error;
|
|
347
459
|
}
|
|
348
|
-
|
|
349
|
-
|
|
460
|
+
|
|
461
|
+
const formatter = new ErrorFormatter({ verbose: options.verbose });
|
|
462
|
+
console.error(formatter.formatMigrationFailureHeader());
|
|
463
|
+
|
|
464
|
+
// Display detailed error information
|
|
465
|
+
if (error instanceof Error && error.name === 'ValidationError') {
|
|
466
|
+
const validationError = error as ValidationError & {
|
|
467
|
+
validationErrors?: Array<{
|
|
468
|
+
rowIndex: number;
|
|
469
|
+
rowData: unknown;
|
|
470
|
+
pkValue: unknown;
|
|
471
|
+
error: ValidationError;
|
|
472
|
+
}>;
|
|
473
|
+
};
|
|
474
|
+
|
|
475
|
+
// Display all validation errors
|
|
476
|
+
if (validationError.validationErrors) {
|
|
477
|
+
console.error(
|
|
478
|
+
`\nFound ${validationError.validationErrors.length} validation error(s) in transformed data:\n`,
|
|
479
|
+
);
|
|
480
|
+
|
|
481
|
+
const errorInfos = validationError.validationErrors.map(
|
|
482
|
+
({ rowIndex, rowData, error: rowError }) => ({
|
|
483
|
+
file: filePath,
|
|
484
|
+
rowIndex,
|
|
485
|
+
issues: rowError.issues,
|
|
486
|
+
data: rowData,
|
|
487
|
+
originalData: rowsToMigrate[rowIndex],
|
|
488
|
+
}),
|
|
489
|
+
);
|
|
490
|
+
|
|
491
|
+
const formatted = formatter.formatValidationErrors(errorInfos);
|
|
492
|
+
console.error(formatted);
|
|
493
|
+
} else {
|
|
494
|
+
// Fallback for single validation error (backward compatibility)
|
|
495
|
+
console.error('\nValidation error:\n');
|
|
496
|
+
const errorInfo = {
|
|
497
|
+
file: filePath,
|
|
498
|
+
rowIndex: 0,
|
|
499
|
+
issues: validationError.issues,
|
|
500
|
+
};
|
|
501
|
+
const formatted = formatter.formatValidationErrors([errorInfo]);
|
|
502
|
+
console.error(formatted);
|
|
503
|
+
}
|
|
504
|
+
} else if (error instanceof Error) {
|
|
505
|
+
console.error(`\n ${error.message}`);
|
|
506
|
+
|
|
507
|
+
// Output stack trace for debugging
|
|
508
|
+
if (options.verbose && error.stack) {
|
|
509
|
+
console.error(`\nStack trace:\n${error.stack}`);
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
// Check if it's a SQLite constraint error
|
|
513
|
+
if (
|
|
514
|
+
error.message.includes('UNIQUE constraint failed') ||
|
|
515
|
+
error.message.includes('FOREIGN KEY constraint failed') ||
|
|
516
|
+
error.message.includes('NOT NULL constraint failed') ||
|
|
517
|
+
error.message.includes('CHECK constraint failed')
|
|
518
|
+
) {
|
|
519
|
+
console.error('\n This is a SQLite constraint violation.');
|
|
520
|
+
console.error(' Please check your data and schema requirements.');
|
|
521
|
+
}
|
|
522
|
+
} else {
|
|
523
|
+
console.error(`\n ${String(error)}`);
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
console.error('');
|
|
527
|
+
process.exit(1);
|
|
528
|
+
}
|
|
529
|
+
} catch (error) {
|
|
530
|
+
await db.close();
|
|
531
|
+
throw error;
|
|
532
|
+
}
|
|
533
|
+
}
|
|
350
534
|
|
|
351
535
|
program.parse();
|