appwrite-utils-cli 0.9.97 → 0.9.981
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/main.js +17 -17
- package/dist/migrations/dataLoader.d.ts +56 -39
- package/dist/migrations/dataLoader.js +15 -6
- package/dist/migrations/databases.js +2 -0
- package/dist/migrations/importController.js +1 -0
- package/dist/utils/helperFunctions.js +10 -3
- package/dist/utils/loadConfigs.js +4 -0
- package/package.json +1 -1
- package/src/main.ts +25 -24
- package/src/migrations/dataLoader.ts +29 -8
- package/src/migrations/databases.ts +1 -0
- package/src/migrations/importController.ts +1 -0
- package/src/utils/helperFunctions.ts +9 -3
- package/src/utils/loadConfigs.ts +4 -0
- package/src/utilsController.ts +0 -1
- package/tsconfig.json +1 -1
- package/dist/appwrite/appwriteConfig.d.ts +0 -3
- package/dist/appwrite/appwriteConfig.js +0 -40
- package/dist/appwrite/collections/Album.d.ts +0 -3
- package/dist/appwrite/collections/Album.js +0 -58
- package/dist/appwrite/collections/Artist.d.ts +0 -3
- package/dist/appwrite/collections/Artist.js +0 -65
- package/dist/appwrite/collections/Genre.d.ts +0 -3
- package/dist/appwrite/collections/Genre.js +0 -33
- package/dist/appwrite/collections/Playlist.d.ts +0 -3
- package/dist/appwrite/collections/Playlist.js +0 -55
- package/dist/appwrite/collections/Song.d.ts +0 -3
- package/dist/appwrite/collections/Song.js +0 -88
- package/dist/appwrite/collections/UserActivity.d.ts +0 -3
- package/dist/appwrite/collections/UserActivity.js +0 -51
- package/dist/appwrite/collections/UserData.d.ts +0 -3
- package/dist/appwrite/collections/UserData.js +0 -33
- package/dist/appwrite/schemas/album.d.ts +0 -32
- package/dist/appwrite/schemas/album.js +0 -12
- package/dist/appwrite/schemas/artist.d.ts +0 -35
- package/dist/appwrite/schemas/artist.js +0 -13
- package/dist/appwrite/schemas/genre.d.ts +0 -23
- package/dist/appwrite/schemas/genre.js +0 -9
- package/dist/appwrite/schemas/playlist.d.ts +0 -32
- package/dist/appwrite/schemas/playlist.js +0 -12
- package/dist/appwrite/schemas/song.d.ts +0 -47
- package/dist/appwrite/schemas/song.js +0 -17
- package/dist/appwrite/schemas/userActivity.d.ts +0 -29
- package/dist/appwrite/schemas/userActivity.js +0 -11
- package/dist/appwrite/schemas/userData.d.ts +0 -23
- package/dist/appwrite/schemas/userData.js +0 -9
package/README.md
CHANGED
@@ -125,6 +125,8 @@ This updated CLI ensures that developers have robust tools at their fingertips t
|
|
125
125
|
|
126
126
|
## Changelog
|
127
127
|
|
128
|
+
- 0.9.981: Try fixing `tryAwaitWithRetry` to catch `522` errors from Cloudflare, they were appearing for some users, also added a 1000ms delay to `tryAwaitWithRetry`
|
129
|
+
- 0.9.98: Fixing some import errors reported by users
|
128
130
|
- 0.9.95: Updated to include new version of `appwrite-utils`
|
129
131
|
- 0.9.93: Updated `selectDatabases` and `selectCollections` from the interactive CLI to prefer local collections or databases when synchronizing the databases
|
130
132
|
- 0.9.92: Fixed `createOrUpdateAttributes` so it deletes attributes that don't exist in local config when you are running `syncDb`. Also updated the database and collection selection, so it won't try and fetch the collections and databases that don't exist (ones you picked from local config) and error
|
package/dist/main.js
CHANGED
@@ -162,20 +162,9 @@ async function main() {
|
|
162
162
|
options.collections = [];
|
163
163
|
}
|
164
164
|
}
|
165
|
-
if (
|
166
|
-
const
|
167
|
-
|
168
|
-
if (options.collections) {
|
169
|
-
for (const db of databases) {
|
170
|
-
const dbCollections = await fetchAllCollections(db.$id, controller.database);
|
171
|
-
collections = collections.concat(dbCollections.filter((c) => options.collections.includes(c.$id)));
|
172
|
-
}
|
173
|
-
}
|
174
|
-
if (parsedArgv.push) {
|
175
|
-
await controller.syncDb(databases, collections);
|
176
|
-
}
|
177
|
-
else if (parsedArgv.sync) {
|
178
|
-
await controller.synchronizeConfigurations(databases);
|
165
|
+
if (options.doBackup && options.databases) {
|
166
|
+
for (const db of options.databases) {
|
167
|
+
await controller.backupDatabase(db);
|
179
168
|
}
|
180
169
|
}
|
181
170
|
if (options.wipeDatabase ||
|
@@ -205,9 +194,20 @@ async function main() {
|
|
205
194
|
}
|
206
195
|
}
|
207
196
|
}
|
208
|
-
if (
|
209
|
-
|
210
|
-
|
197
|
+
if (parsedArgv.push || parsedArgv.sync) {
|
198
|
+
const databases = options.databases || (await fetchAllDatabases(controller.database));
|
199
|
+
let collections = [];
|
200
|
+
if (options.collections) {
|
201
|
+
for (const db of databases) {
|
202
|
+
const dbCollections = await fetchAllCollections(db.$id, controller.database);
|
203
|
+
collections = collections.concat(dbCollections.filter((c) => options.collections.includes(c.$id)));
|
204
|
+
}
|
205
|
+
}
|
206
|
+
if (parsedArgv.push) {
|
207
|
+
await controller.syncDb(databases, collections);
|
208
|
+
}
|
209
|
+
else if (parsedArgv.sync) {
|
210
|
+
await controller.synchronizeConfigurations(databases);
|
211
211
|
}
|
212
212
|
}
|
213
213
|
if (options.generateSchemas) {
|
@@ -79,7 +79,17 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
|
|
79
79
|
key: string;
|
80
80
|
type?: "integer" | undefined;
|
81
81
|
description?: string | Record<string, string> | null | undefined;
|
82
|
-
required
|
82
|
+
required
|
83
|
+
/**
|
84
|
+
* Prepares the data for creating user collection documents.
|
85
|
+
* This involves loading the data, transforming it according to the import definition,
|
86
|
+
* and handling the creation of new unique IDs for each item.
|
87
|
+
*
|
88
|
+
* @param db - The database configuration.
|
89
|
+
* @param collection - The collection configuration.
|
90
|
+
* @param importDef - The import definition containing the attribute mappings and other relevant info.
|
91
|
+
*/
|
92
|
+
?: boolean | undefined;
|
83
93
|
array?: boolean | undefined;
|
84
94
|
error?: string | undefined;
|
85
95
|
xdefault?: number | null | undefined;
|
@@ -674,18 +684,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
|
|
674
684
|
}[] | undefined;
|
675
685
|
}[];
|
676
686
|
type?: "create" | "update" | undefined;
|
677
|
-
basePath
|
678
|
-
/**
|
679
|
-
* Generates attribute mappings with post-import actions based on the provided attribute mappings.
|
680
|
-
* This method checks each mapping for a fileData attribute and adds a post-import action to create a file
|
681
|
-
* and update the field with the file's ID if necessary.
|
682
|
-
*
|
683
|
-
* @param attributeMappings - The attribute mappings from the import definition.
|
684
|
-
* @param context - The context object containing information about the database, collection, and document.
|
685
|
-
* @param item - The item being imported, used for resolving template paths in fileData mappings.
|
686
|
-
* @returns The attribute mappings updated with any necessary post-import actions.
|
687
|
-
*/
|
688
|
-
?: string | undefined;
|
687
|
+
basePath?: string | undefined;
|
689
688
|
idMappings?: {
|
690
689
|
sourceField: string;
|
691
690
|
targetField: string;
|
@@ -709,7 +708,27 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
|
|
709
708
|
attributes?: ({
|
710
709
|
key: string;
|
711
710
|
type?: "string" | undefined;
|
712
|
-
format
|
711
|
+
format
|
712
|
+
/**
|
713
|
+
* Generates attribute mappings with post-import actions based on the provided attribute mappings.
|
714
|
+
* This method checks each mapping for a fileData attribute and adds a post-import action to create a file
|
715
|
+
* and update the field with the file's ID if necessary.
|
716
|
+
*
|
717
|
+
* @param attributeMappings - The attribute mappings from the import definition.
|
718
|
+
* @param context - The context object containing information about the database, collection, and document.
|
719
|
+
* @param item - The item being imported, used for resolving template paths in fileData mappings.
|
720
|
+
* @returns The attribute mappings updated with any necessary post-import actions.
|
721
|
+
*/
|
722
|
+
? /**
|
723
|
+
* Generates attribute mappings with post-import actions based on the provided attribute mappings.
|
724
|
+
* This method checks each mapping for a fileData attribute and adds a post-import action to create a file
|
725
|
+
* and update the field with the file's ID if necessary.
|
726
|
+
*
|
727
|
+
* @param attributeMappings - The attribute mappings from the import definition.
|
728
|
+
* @param context - The context object containing information about the database, collection, and document.
|
729
|
+
* @param item - The item being imported, used for resolving template paths in fileData mappings.
|
730
|
+
* @returns The attribute mappings updated with any necessary post-import actions.
|
731
|
+
*/: string | null | undefined;
|
713
732
|
description?: string | Record<string, string> | undefined;
|
714
733
|
required?: boolean | undefined;
|
715
734
|
array?: boolean | undefined;
|
@@ -1305,18 +1324,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
|
|
1305
1324
|
}[] | undefined;
|
1306
1325
|
}[];
|
1307
1326
|
type?: "create" | "update" | undefined;
|
1308
|
-
basePath
|
1309
|
-
/**
|
1310
|
-
* Generates attribute mappings with post-import actions based on the provided attribute mappings.
|
1311
|
-
* This method checks each mapping for a fileData attribute and adds a post-import action to create a file
|
1312
|
-
* and update the field with the file's ID if necessary.
|
1313
|
-
*
|
1314
|
-
* @param attributeMappings - The attribute mappings from the import definition.
|
1315
|
-
* @param context - The context object containing information about the database, collection, and document.
|
1316
|
-
* @param item - The item being imported, used for resolving template paths in fileData mappings.
|
1317
|
-
* @returns The attribute mappings updated with any necessary post-import actions.
|
1318
|
-
*/
|
1319
|
-
?: string | undefined;
|
1327
|
+
basePath?: string | undefined;
|
1320
1328
|
idMappings?: {
|
1321
1329
|
sourceField: string;
|
1322
1330
|
targetField: string;
|
@@ -1384,7 +1392,27 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
|
|
1384
1392
|
attributes?: ({
|
1385
1393
|
key: string;
|
1386
1394
|
type?: "string" | undefined;
|
1387
|
-
format
|
1395
|
+
format
|
1396
|
+
/**
|
1397
|
+
* Generates attribute mappings with post-import actions based on the provided attribute mappings.
|
1398
|
+
* This method checks each mapping for a fileData attribute and adds a post-import action to create a file
|
1399
|
+
* and update the field with the file's ID if necessary.
|
1400
|
+
*
|
1401
|
+
* @param attributeMappings - The attribute mappings from the import definition.
|
1402
|
+
* @param context - The context object containing information about the database, collection, and document.
|
1403
|
+
* @param item - The item being imported, used for resolving template paths in fileData mappings.
|
1404
|
+
* @returns The attribute mappings updated with any necessary post-import actions.
|
1405
|
+
*/
|
1406
|
+
? /**
|
1407
|
+
* Generates attribute mappings with post-import actions based on the provided attribute mappings.
|
1408
|
+
* This method checks each mapping for a fileData attribute and adds a post-import action to create a file
|
1409
|
+
* and update the field with the file's ID if necessary.
|
1410
|
+
*
|
1411
|
+
* @param attributeMappings - The attribute mappings from the import definition.
|
1412
|
+
* @param context - The context object containing information about the database, collection, and document.
|
1413
|
+
* @param item - The item being imported, used for resolving template paths in fileData mappings.
|
1414
|
+
* @returns The attribute mappings updated with any necessary post-import actions.
|
1415
|
+
*/: string | null | undefined;
|
1388
1416
|
description?: string | Record<string, string> | undefined;
|
1389
1417
|
required?: boolean | undefined;
|
1390
1418
|
array?: boolean | undefined;
|
@@ -1720,18 +1748,7 @@ export declare class DataLoader {
|
|
1720
1748
|
}[] | undefined;
|
1721
1749
|
}[];
|
1722
1750
|
type?: "create" | "update" | undefined;
|
1723
|
-
basePath
|
1724
|
-
/**
|
1725
|
-
* Generates attribute mappings with post-import actions based on the provided attribute mappings.
|
1726
|
-
* This method checks each mapping for a fileData attribute and adds a post-import action to create a file
|
1727
|
-
* and update the field with the file's ID if necessary.
|
1728
|
-
*
|
1729
|
-
* @param attributeMappings - The attribute mappings from the import definition.
|
1730
|
-
* @param context - The context object containing information about the database, collection, and document.
|
1731
|
-
* @param item - The item being imported, used for resolving template paths in fileData mappings.
|
1732
|
-
* @returns The attribute mappings updated with any necessary post-import actions.
|
1733
|
-
*/
|
1734
|
-
?: string | undefined;
|
1751
|
+
basePath?: string | undefined;
|
1735
1752
|
idMappings?: {
|
1736
1753
|
sourceField: string;
|
1737
1754
|
targetField: string;
|
@@ -147,17 +147,20 @@ export class DataLoader {
|
|
147
147
|
}
|
148
148
|
// Method to load data from a file specified in the import definition
|
149
149
|
loadData(importDef) {
|
150
|
-
//
|
150
|
+
// Simply join appwriteFolderPath with the importDef.filePath
|
151
151
|
const filePath = path.resolve(this.appwriteFolderPath, importDef.filePath);
|
152
|
+
console.log(`Loading data from: ${filePath}`); // Add logging
|
152
153
|
if (!fs.existsSync(filePath)) {
|
153
154
|
console.error(`File not found: ${filePath}`);
|
154
155
|
return [];
|
155
156
|
}
|
156
157
|
// Read the file and parse the JSON data
|
157
158
|
const rawData = fs.readFileSync(filePath, "utf8");
|
158
|
-
|
159
|
+
const parsedData = importDef.basePath
|
159
160
|
? JSON.parse(rawData)[importDef.basePath]
|
160
161
|
: JSON.parse(rawData);
|
162
|
+
console.log(`Loaded ${parsedData?.length || 0} items from ${filePath}`);
|
163
|
+
return parsedData;
|
161
164
|
}
|
162
165
|
// Helper method to check if a new ID already exists in the old-to-new ID map
|
163
166
|
checkMapValuesForId(newId, collectionName) {
|
@@ -739,7 +742,7 @@ export class DataLoader {
|
|
739
742
|
async prepareUserCollectionCreateData(db, collection, importDef) {
|
740
743
|
// Load the raw data based on the import definition
|
741
744
|
const rawData = this.loadData(importDef);
|
742
|
-
|
745
|
+
let operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
|
743
746
|
// Initialize a new map for old ID to new ID mappings
|
744
747
|
const oldIdToNewIdMap = new Map();
|
745
748
|
// Retrieve or initialize the collection-specific old ID to new ID map
|
@@ -748,7 +751,10 @@ export class DataLoader {
|
|
748
751
|
.set(this.getCollectionKey(collection.name), oldIdToNewIdMap)
|
749
752
|
.get(this.getCollectionKey(collection.name));
|
750
753
|
if (!operationId) {
|
751
|
-
|
754
|
+
const collectionImportOperation = await findOrCreateOperation(this.database, collection.$id, "importData");
|
755
|
+
// Store the operation ID in the map
|
756
|
+
this.collectionImportOperations.set(this.getCollectionKey(collection.name), collectionImportOperation.$id);
|
757
|
+
operationId = collectionImportOperation.$id;
|
752
758
|
}
|
753
759
|
await updateOperation(this.database, operationId, {
|
754
760
|
status: "ready",
|
@@ -892,9 +898,12 @@ export class DataLoader {
|
|
892
898
|
async prepareCreateData(db, collection, importDef) {
|
893
899
|
// Load the raw data based on the import definition
|
894
900
|
const rawData = this.loadData(importDef);
|
895
|
-
|
901
|
+
let operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
|
896
902
|
if (!operationId) {
|
897
|
-
|
903
|
+
const collectionImportOperation = await findOrCreateOperation(this.database, collection.$id, "importData");
|
904
|
+
// Store the operation ID in the map
|
905
|
+
this.collectionImportOperations.set(this.getCollectionKey(collection.name), collectionImportOperation.$id);
|
906
|
+
operationId = collectionImportOperation.$id;
|
898
907
|
}
|
899
908
|
await updateOperation(this.database, operationId, {
|
900
909
|
status: "ready",
|
@@ -2,6 +2,8 @@ import { Client, Databases, IndexType, Query, } from "node-appwrite";
|
|
2
2
|
import { getAppwriteClient, tryAwaitWithRetry, } from "../utils/helperFunctions.js";
|
3
3
|
export const fetchAllDatabases = async (database) => {
|
4
4
|
const databases = await tryAwaitWithRetry(async () => await database.list([Query.limit(25)]));
|
5
|
+
if (!databases)
|
6
|
+
return [];
|
5
7
|
const allDatabases = databases.databases;
|
6
8
|
if (allDatabases.length === 0)
|
7
9
|
return [];
|
@@ -53,6 +53,7 @@ export class ImportController {
|
|
53
53
|
if (!databaseRan) {
|
54
54
|
databaseRan = db;
|
55
55
|
dataLoader = new DataLoader(this.appwriteFolderPath, this.importDataActions, this.database, this.config, this.setupOptions.shouldWriteFile);
|
56
|
+
await dataLoader.setupMaps(db.$id);
|
56
57
|
await dataLoader.start(db.$id);
|
57
58
|
await this.importCollections(db, dataLoader, specificCollections);
|
58
59
|
await resolveAndUpdateRelationships(db.$id, this.database, this.config);
|
@@ -92,14 +92,21 @@ export const tryAwaitWithRetry = async (createFunction, attemptNum = 0, throwErr
|
|
92
92
|
return await createFunction();
|
93
93
|
}
|
94
94
|
catch (error) {
|
95
|
-
if (error instanceof AppwriteException &&
|
95
|
+
if ((error instanceof AppwriteException &&
|
96
96
|
(error.message.toLowerCase().includes("fetch failed") ||
|
97
|
-
error.message.toLowerCase().includes("server error")))
|
97
|
+
error.message.toLowerCase().includes("server error"))) ||
|
98
|
+
(error.code === 522 || error.code === "522")) {
|
99
|
+
if (error.code === 522) {
|
100
|
+
console.log("Cloudflare error. Retrying...");
|
101
|
+
}
|
102
|
+
else {
|
103
|
+
console.log(`Fetch failed on attempt ${attemptNum}. Retrying...`);
|
104
|
+
}
|
98
105
|
numTimesFailedTotal++;
|
99
|
-
console.log(`Fetch failed on attempt ${attemptNum}. Retrying...`);
|
100
106
|
if (attemptNum > 5) {
|
101
107
|
throw error;
|
102
108
|
}
|
109
|
+
await delay(1000);
|
103
110
|
return tryAwaitWithRetry(createFunction, attemptNum + 1);
|
104
111
|
}
|
105
112
|
if (throwError) {
|
@@ -53,6 +53,10 @@ export const loadConfig = async (configDir) => {
|
|
53
53
|
const collectionModule = (await import(fileUrl));
|
54
54
|
const collection = collectionModule.default?.default || collectionModule.default || collectionModule;
|
55
55
|
if (collection) {
|
56
|
+
// Ensure importDefs are properly loaded
|
57
|
+
if (collectionModule.importDefs || collection.importDefs) {
|
58
|
+
collection.importDefs = collectionModule.importDefs || collection.importDefs;
|
59
|
+
}
|
56
60
|
config.collections.push(collection);
|
57
61
|
}
|
58
62
|
}
|
package/package.json
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
{
|
2
2
|
"name": "appwrite-utils-cli",
|
3
3
|
"description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
|
4
|
-
"version": "0.9.
|
4
|
+
"version": "0.9.981",
|
5
5
|
"main": "src/main.ts",
|
6
6
|
"type": "module",
|
7
7
|
"repository": {
|
package/src/main.ts
CHANGED
@@ -206,27 +206,9 @@ async function main() {
|
|
206
206
|
}
|
207
207
|
}
|
208
208
|
|
209
|
-
if (
|
210
|
-
const databases
|
211
|
-
|
212
|
-
let collections: Models.Collection[] = [];
|
213
|
-
|
214
|
-
if (options.collections) {
|
215
|
-
for (const db of databases) {
|
216
|
-
const dbCollections = await fetchAllCollections(
|
217
|
-
db.$id,
|
218
|
-
controller.database!
|
219
|
-
);
|
220
|
-
collections = collections.concat(
|
221
|
-
dbCollections.filter((c) => options.collections!.includes(c.$id))
|
222
|
-
);
|
223
|
-
}
|
224
|
-
}
|
225
|
-
|
226
|
-
if (parsedArgv.push) {
|
227
|
-
await controller.syncDb(databases, collections);
|
228
|
-
} else if (parsedArgv.sync) {
|
229
|
-
await controller.synchronizeConfigurations(databases);
|
209
|
+
if (options.doBackup && options.databases) {
|
210
|
+
for (const db of options.databases) {
|
211
|
+
await controller.backupDatabase(db);
|
230
212
|
}
|
231
213
|
}
|
232
214
|
|
@@ -265,12 +247,31 @@ async function main() {
|
|
265
247
|
}
|
266
248
|
}
|
267
249
|
|
268
|
-
if (
|
269
|
-
|
270
|
-
await controller.
|
250
|
+
if (parsedArgv.push || parsedArgv.sync) {
|
251
|
+
const databases =
|
252
|
+
options.databases || (await fetchAllDatabases(controller.database!));
|
253
|
+
let collections: Models.Collection[] = [];
|
254
|
+
|
255
|
+
if (options.collections) {
|
256
|
+
for (const db of databases) {
|
257
|
+
const dbCollections = await fetchAllCollections(
|
258
|
+
db.$id,
|
259
|
+
controller.database!
|
260
|
+
);
|
261
|
+
collections = collections.concat(
|
262
|
+
dbCollections.filter((c) => options.collections!.includes(c.$id))
|
263
|
+
);
|
264
|
+
}
|
265
|
+
}
|
266
|
+
|
267
|
+
if (parsedArgv.push) {
|
268
|
+
await controller.syncDb(databases, collections);
|
269
|
+
} else if (parsedArgv.sync) {
|
270
|
+
await controller.synchronizeConfigurations(databases);
|
271
271
|
}
|
272
272
|
}
|
273
273
|
|
274
|
+
|
274
275
|
if (options.generateSchemas) {
|
275
276
|
await controller.generateSchemas();
|
276
277
|
}
|
@@ -24,6 +24,7 @@ import { AuthUserCreateSchema } from "../schemas/authUser.js";
|
|
24
24
|
import _ from "lodash";
|
25
25
|
import { UsersController } from "./users.js";
|
26
26
|
import { finalizeByAttributeMap } from "../utils/helperFunctions.js";
|
27
|
+
|
27
28
|
// Define a schema for the structure of collection import data using Zod for validation
|
28
29
|
export const CollectionImportDataSchema = z.object({
|
29
30
|
// Optional collection creation schema
|
@@ -182,8 +183,9 @@ export class DataLoader {
|
|
182
183
|
|
183
184
|
// Method to load data from a file specified in the import definition
|
184
185
|
loadData(importDef: ImportDef): any[] {
|
185
|
-
//
|
186
|
+
// Simply join appwriteFolderPath with the importDef.filePath
|
186
187
|
const filePath = path.resolve(this.appwriteFolderPath, importDef.filePath);
|
188
|
+
console.log(`Loading data from: ${filePath}`); // Add logging
|
187
189
|
if (!fs.existsSync(filePath)) {
|
188
190
|
console.error(`File not found: ${filePath}`);
|
189
191
|
return [];
|
@@ -191,9 +193,12 @@ export class DataLoader {
|
|
191
193
|
|
192
194
|
// Read the file and parse the JSON data
|
193
195
|
const rawData = fs.readFileSync(filePath, "utf8");
|
194
|
-
|
196
|
+
const parsedData = importDef.basePath
|
195
197
|
? JSON.parse(rawData)[importDef.basePath]
|
196
198
|
: JSON.parse(rawData);
|
199
|
+
|
200
|
+
console.log(`Loaded ${parsedData?.length || 0} items from ${filePath}`);
|
201
|
+
return parsedData;
|
197
202
|
}
|
198
203
|
|
199
204
|
// Helper method to check if a new ID already exists in the old-to-new ID map
|
@@ -932,7 +937,7 @@ export class DataLoader {
|
|
932
937
|
): Promise<void> {
|
933
938
|
// Load the raw data based on the import definition
|
934
939
|
const rawData = this.loadData(importDef);
|
935
|
-
|
940
|
+
let operationId = this.collectionImportOperations.get(
|
936
941
|
this.getCollectionKey(collection.name)
|
937
942
|
);
|
938
943
|
// Initialize a new map for old ID to new ID mappings
|
@@ -946,9 +951,17 @@ export class DataLoader {
|
|
946
951
|
.set(this.getCollectionKey(collection.name), oldIdToNewIdMap)
|
947
952
|
.get(this.getCollectionKey(collection.name));
|
948
953
|
if (!operationId) {
|
949
|
-
|
950
|
-
|
954
|
+
const collectionImportOperation = await findOrCreateOperation(
|
955
|
+
this.database,
|
956
|
+
collection.$id,
|
957
|
+
"importData"
|
951
958
|
);
|
959
|
+
// Store the operation ID in the map
|
960
|
+
this.collectionImportOperations.set(
|
961
|
+
this.getCollectionKey(collection.name),
|
962
|
+
collectionImportOperation.$id
|
963
|
+
);
|
964
|
+
operationId = collectionImportOperation.$id;
|
952
965
|
}
|
953
966
|
await updateOperation(this.database, operationId, {
|
954
967
|
status: "ready",
|
@@ -1160,13 +1173,21 @@ export class DataLoader {
|
|
1160
1173
|
): Promise<void> {
|
1161
1174
|
// Load the raw data based on the import definition
|
1162
1175
|
const rawData = this.loadData(importDef);
|
1163
|
-
|
1176
|
+
let operationId = this.collectionImportOperations.get(
|
1164
1177
|
this.getCollectionKey(collection.name)
|
1165
1178
|
);
|
1166
1179
|
if (!operationId) {
|
1167
|
-
|
1168
|
-
|
1180
|
+
const collectionImportOperation = await findOrCreateOperation(
|
1181
|
+
this.database,
|
1182
|
+
collection.$id,
|
1183
|
+
"importData"
|
1184
|
+
);
|
1185
|
+
// Store the operation ID in the map
|
1186
|
+
this.collectionImportOperations.set(
|
1187
|
+
this.getCollectionKey(collection.name),
|
1188
|
+
collectionImportOperation.$id
|
1169
1189
|
);
|
1190
|
+
operationId = collectionImportOperation.$id;
|
1170
1191
|
}
|
1171
1192
|
await updateOperation(this.database, operationId, {
|
1172
1193
|
status: "ready",
|
@@ -16,6 +16,7 @@ export const fetchAllDatabases = async (
|
|
16
16
|
const databases = await tryAwaitWithRetry(
|
17
17
|
async () => await database.list([Query.limit(25)])
|
18
18
|
);
|
19
|
+
if (!databases) return [];
|
19
20
|
const allDatabases = databases.databases;
|
20
21
|
if (allDatabases.length === 0) return [];
|
21
22
|
let lastDatabaseId = allDatabases[allDatabases.length - 1].$id;
|
@@ -100,6 +100,7 @@ export class ImportController {
|
|
100
100
|
this.config,
|
101
101
|
this.setupOptions.shouldWriteFile
|
102
102
|
);
|
103
|
+
await dataLoader.setupMaps(db.$id);
|
103
104
|
await dataLoader.start(db.$id);
|
104
105
|
await this.importCollections(db, dataLoader, specificCollections);
|
105
106
|
await resolveAndUpdateRelationships(db.$id, this.database, this.config);
|
@@ -149,15 +149,21 @@ export const tryAwaitWithRetry = async <T>(
|
|
149
149
|
return await createFunction();
|
150
150
|
} catch (error) {
|
151
151
|
if (
|
152
|
-
error instanceof AppwriteException &&
|
152
|
+
(error instanceof AppwriteException &&
|
153
153
|
(error.message.toLowerCase().includes("fetch failed") ||
|
154
|
-
error.message.toLowerCase().includes("server error"))
|
154
|
+
error.message.toLowerCase().includes("server error"))) ||
|
155
|
+
((error as any).code === 522 || (error as any).code === "522")
|
155
156
|
) {
|
157
|
+
if ((error as any).code === 522) {
|
158
|
+
console.log("Cloudflare error. Retrying...");
|
159
|
+
} else {
|
160
|
+
console.log(`Fetch failed on attempt ${attemptNum}. Retrying...`);
|
161
|
+
}
|
156
162
|
numTimesFailedTotal++;
|
157
|
-
console.log(`Fetch failed on attempt ${attemptNum}. Retrying...`);
|
158
163
|
if (attemptNum > 5) {
|
159
164
|
throw error;
|
160
165
|
}
|
166
|
+
await delay(1000);
|
161
167
|
return tryAwaitWithRetry(createFunction, attemptNum + 1);
|
162
168
|
}
|
163
169
|
if (throwError) {
|
package/src/utils/loadConfigs.ts
CHANGED
@@ -60,6 +60,10 @@ export const loadConfig = async (
|
|
60
60
|
const collectionModule = (await import(fileUrl));
|
61
61
|
const collection: Collection | undefined = collectionModule.default?.default || collectionModule.default || collectionModule;
|
62
62
|
if (collection) {
|
63
|
+
// Ensure importDefs are properly loaded
|
64
|
+
if (collectionModule.importDefs || collection.importDefs) {
|
65
|
+
collection.importDefs = collectionModule.importDefs || collection.importDefs;
|
66
|
+
}
|
63
67
|
config.collections.push(collection);
|
64
68
|
}
|
65
69
|
}
|
package/src/utilsController.ts
CHANGED
package/tsconfig.json
CHANGED
@@ -1,40 +0,0 @@
|
|
1
|
-
import {} from "appwrite-utils";
|
2
|
-
const appwriteConfig = {
|
3
|
-
appwriteEndpoint: "https://appwrite.blackleafdigital.com/v1",
|
4
|
-
appwriteProject: "6717441100056a90bb20",
|
5
|
-
appwriteKey: "standard_f8664be4666240926caa1a2ed15a87424948036e3fd47b8d99a3de0cad83cdea7309ae30d004a3edd44c45292605f4a6b9c654aa44924a7cc5e3e95a77e0c9513d9c29a9059b7cf4fdafe626b67147d26de78390f946aeca22328f7ebdcdad159a644eb7589cacecdf4e397bd51362230cd319544a804db62d8edb968de1392a",
|
6
|
-
enableBackups: true,
|
7
|
-
backupInterval: 3600,
|
8
|
-
backupRetention: 30,
|
9
|
-
enableBackupCleanup: true,
|
10
|
-
enableMockData: false,
|
11
|
-
documentBucketId: "documents",
|
12
|
-
usersCollectionName: "Members",
|
13
|
-
databases: [
|
14
|
-
{
|
15
|
-
$id: "main",
|
16
|
-
name: "Main",
|
17
|
-
bucket: {
|
18
|
-
$id: "main_bucket",
|
19
|
-
name: "Main Bucket",
|
20
|
-
enabled: true,
|
21
|
-
maximumFileSize: 30000000,
|
22
|
-
allowedFileExtensions: [],
|
23
|
-
encryption: true,
|
24
|
-
antivirus: true
|
25
|
-
}
|
26
|
-
},
|
27
|
-
],
|
28
|
-
buckets: [
|
29
|
-
{
|
30
|
-
$id: "global_bucket",
|
31
|
-
name: "Global Bucket",
|
32
|
-
enabled: true,
|
33
|
-
maximumFileSize: 30000000,
|
34
|
-
allowedFileExtensions: [],
|
35
|
-
encryption: true,
|
36
|
-
antivirus: true
|
37
|
-
}
|
38
|
-
]
|
39
|
-
};
|
40
|
-
export default appwriteConfig;
|
@@ -1,58 +0,0 @@
|
|
1
|
-
const Album = {
|
2
|
-
$id: '01JASDQZ5TXZZ1Z9EVR5PKC10G',
|
3
|
-
documentSecurity: false,
|
4
|
-
enabled: true,
|
5
|
-
name: 'Album',
|
6
|
-
$permissions: [
|
7
|
-
{ permission: 'read', target: 'any' },
|
8
|
-
{ permission: 'create', target: 'users' },
|
9
|
-
{ permission: 'update', target: 'users' },
|
10
|
-
{ permission: 'delete', target: 'users' }
|
11
|
-
],
|
12
|
-
attributes: [
|
13
|
-
{
|
14
|
-
key: 'title',
|
15
|
-
type: 'string',
|
16
|
-
size: 256,
|
17
|
-
required: true,
|
18
|
-
},
|
19
|
-
{
|
20
|
-
key: 'artistId',
|
21
|
-
type: 'string',
|
22
|
-
size: 36,
|
23
|
-
required: true,
|
24
|
-
},
|
25
|
-
{
|
26
|
-
key: 'releaseDate',
|
27
|
-
type: 'float',
|
28
|
-
min: 0,
|
29
|
-
required: true,
|
30
|
-
},
|
31
|
-
{
|
32
|
-
key: 'coverArtId',
|
33
|
-
type: 'string',
|
34
|
-
size: 36,
|
35
|
-
required: false,
|
36
|
-
},
|
37
|
-
{
|
38
|
-
key: 'genreIds',
|
39
|
-
type: 'string',
|
40
|
-
size: 36,
|
41
|
-
array: true,
|
42
|
-
required: false,
|
43
|
-
},
|
44
|
-
],
|
45
|
-
indexes: [
|
46
|
-
{
|
47
|
-
key: 'title',
|
48
|
-
type: 'fulltext',
|
49
|
-
attributes: ['title'],
|
50
|
-
},
|
51
|
-
{
|
52
|
-
key: 'artistId',
|
53
|
-
type: 'key',
|
54
|
-
attributes: ['artistId'],
|
55
|
-
},
|
56
|
-
],
|
57
|
-
};
|
58
|
-
export default Album;
|