@abtnode/core 1.16.47-beta-20250721-073316-a6de1fa3 → 1.16.47-beta-20250721-130532-61549a96

package/lib/util/migration-sqlite-to-postgres.js

@@ -10,6 +10,9 @@ const path = require('path');
 
 const ignoreErrorTableNames = new Set(['runtime_insights', 'notification_receivers', 'notifications']);
 const needCleanDataTableNames = new Set(['sessions']);
+const notCheckPrimaryKeyTableNames = new Set(['tagging']);
+
+const needBreakErrors = [];
 
 function sortTableNames(tableNames, sort) {
   return [...tableNames].sort((a, b) => {
@@ -49,10 +52,8 @@ async function migrateAllTablesNoModels(dbPath) {
   // 把 tableNames 排序, 把被依赖的表放前面
   tableNames = sortTableNames(tableNames, ['users', 'notification_receivers']);
 
-  console.log('Start migration database: ', dbPath);
-
   for (const tableName of tableNames) {
-    console.log(`\n➡️ Starting migration for table: ${tableName}`);
+    console.log(`\n➡️ Starting migration for table: ${dbPath} ${tableName}`);
 
     const colInfos = await sqliteDb.query(`PRAGMA TABLE_INFO("${tableName}")`, { type: QueryTypes.SELECT });
     const sqliteSchema = {};
@@ -80,6 +81,14 @@ async function migrateAllTablesNoModels(dbPath) {
     // Describe PG table to detect JSON/auto-inc
     const pgSchema = await pgQI.describeTable(tableName);
 
+    const varcharLimits = {};
+    for (const [col, def] of Object.entries(pgSchema)) {
+      const m = def.type.match(/character varying\((\d+)\)/i);
+      if (m) {
+        varcharLimits[col] = parseInt(m[1], 10);
+      }
+    }
+
     // find JSON/JSONB
     const jsonCols = Object.entries(pgSchema)
       .filter(([, def]) => def.type && ['JSON', 'JSONB'].includes(def.type.toUpperCase()))
@@ -115,12 +124,22 @@ async function migrateAllTablesNoModels(dbPath) {
           return jc ? `"${c}" = EXCLUDED."${c}"::${jc.type.toLowerCase()}` : `"${c}" = EXCLUDED."${c}"`;
         })
         .join(',');
-      upsertSQL = `
+      if (notCheckPrimaryKeyTableNames.has(tableName)) {
+        console.log('   ❌ Not check primary key for, can not upsert only insert:', tableName);
+        // 只做普通插入
+        upsertSQL = `
+            INSERT INTO "${tableName}" (${insertColsList})
+            VALUES (${placeholders});
+          `;
+      } else {
+        // 正常的 upsert 逻辑
+        upsertSQL = `
         INSERT INTO "${tableName}" (${insertColsList})
         VALUES (${placeholders})
         ON CONFLICT (${conflictKeys})
         DO UPDATE SET ${updateSet};
       `;
+      }
     } else {
       upsertSQL = `
         INSERT INTO "${tableName}" (${insertColsList})
@@ -153,6 +172,18 @@ async function migrateAllTablesNoModels(dbPath) {
           row.feedType = '';
         }
 
+        // 对所有需插入的列, 操过长度的做截断
+        for (const col of insertCols) {
+          const val = row[col];
+          const limit = varcharLimits[col];
+          if (typeof val === 'string' && limit && val.length > limit) {
+            console.warn(`⚠️ Truncate "${col}" from length ${val.length} → ${limit}`);
+            console.log('   ❌ Old value:', val);
+            // 暂时不截断, 这样插入会失败, 后续会忽略这条数据的插入
+            // row[col] = val.slice(0, limit);
+          }
+        }
+
         for (const jc of jsonCols) {
           const raw = row[jc.name];
           let parsed = null;
@@ -265,7 +296,34 @@ async function migrateAllTablesNoModels(dbPath) {
             }
             continue;
           }
-          throw err;
+          if (err.message.includes('user_sessions_userDid_fkey')) {
+            const violationFilePath = dbPath.replace('.db', '.user_sessions_userDid_fkey.json');
+            console.log(
+              '   ❌ Ignore error for user_sessions_userDid_fkey',
+              err.message,
+              'save to:',
+              violationFilePath,
+              'count: '
+            );
+            try {
+              await fsp.appendFile(
+                violationFilePath,
+                // eslint-disable-next-line prefer-template
+                JSON.stringify({ table: tableName, row, error: 'user_sessions_userDid_fkey' }) + '\n',
+                'utf8'
+              );
+            } catch (writeErr) {
+              console.warn('   ⚠️ Failed to write violation record:', writeErr);
+            }
+            continue;
+          }
+
+          if (needBreakErrors.length < 2000) {
+            console.error('   ❌ Error:', err.message);
+            needBreakErrors.push(err.message);
+          } else {
+            throw err;
+          }
         }
       }
 
@@ -418,6 +476,12 @@ async function migrationSqliteToPostgres(dataDir, dbPaths) {
     await validateTableRowCounts(dbPath);
   }
 
+  if (needBreakErrors.length > 0) {
+    console.error('   ❌ Has some errors, please check the violation files');
+    console.error(needBreakErrors.join('\n'));
+    throw new Error('Has some errors, please check the error log');
+  }
+
   savePostgresLock(dataDir);
 }
 

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.16.47-beta-20250721-073316-a6de1fa3",
+  "version": "1.16.47-beta-20250721-130532-61549a96",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,22 +19,22 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "Apache-2.0",
   "dependencies": {
-    "@abtnode/analytics": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/auth": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/certificate-manager": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/client": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/constant": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/cron": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/db-cache": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/docker-utils": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/logger": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/models": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/queue": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/rbac": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/router-provider": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/static-server": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/timemachine": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@abtnode/util": "1.16.47-beta-20250721-073316-a6de1fa3",
+    "@abtnode/analytics": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/auth": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/certificate-manager": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/client": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/constant": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/cron": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/db-cache": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/docker-utils": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/logger": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/models": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/queue": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/rbac": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/router-provider": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/static-server": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/timemachine": "1.16.47-beta-20250721-130532-61549a96",
+    "@abtnode/util": "1.16.47-beta-20250721-130532-61549a96",
     "@arcblock/did": "1.20.16",
     "@arcblock/did-auth": "1.20.16",
     "@arcblock/did-ext": "1.20.16",
@@ -45,14 +45,14 @@
     "@arcblock/pm2-events": "^0.0.5",
     "@arcblock/validator": "1.20.16",
     "@arcblock/vc": "1.20.16",
-    "@blocklet/constant": "1.16.47-beta-20250721-073316-a6de1fa3",
+    "@blocklet/constant": "1.16.47-beta-20250721-130532-61549a96",
     "@blocklet/did-space-js": "^1.1.7",
-    "@blocklet/env": "1.16.47-beta-20250721-073316-a6de1fa3",
+    "@blocklet/env": "1.16.47-beta-20250721-130532-61549a96",
     "@blocklet/error": "^0.2.5",
-    "@blocklet/meta": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@blocklet/resolver": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@blocklet/sdk": "1.16.47-beta-20250721-073316-a6de1fa3",
-    "@blocklet/store": "1.16.47-beta-20250721-073316-a6de1fa3",
+    "@blocklet/meta": "1.16.47-beta-20250721-130532-61549a96",
+    "@blocklet/resolver": "1.16.47-beta-20250721-130532-61549a96",
+    "@blocklet/sdk": "1.16.47-beta-20250721-130532-61549a96",
+    "@blocklet/store": "1.16.47-beta-20250721-130532-61549a96",
     "@blocklet/theme": "^3.0.26",
     "@fidm/x509": "^1.2.1",
     "@ocap/mcrypto": "1.20.16",
@@ -116,5 +116,5 @@
     "jest": "^29.7.0",
     "unzipper": "^0.10.11"
   },
-  "gitHead": "2163d72cfb533e6dbcedd7082c45d7f3dbe3cd90"
+  "gitHead": "78041763148c9b98245c28ff176ff0fc9acbd071"
 }