adula 2.1.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +149 -18
- package/build/agent/AGENTS.template.md +4 -4
- package/build/agent/capabilities.md +22 -12
- package/build/agent/skills/adula-frontend-design/SKILL.md +6 -4
- package/build/agent/skills/idea-review/SKILL.md +2 -2
- package/build/commands/adula_assistant_prune.d.ts +10 -0
- package/build/commands/adula_assistant_prune.js +12 -0
- package/build/commands/adula_outbox.d.ts +10 -0
- package/build/commands/adula_outbox.js +11 -0
- package/build/commands/adula_restore_reconcile.d.ts +10 -0
- package/build/commands/adula_restore_reconcile.js +34 -0
- package/build/commands/adula_runtime_health.d.ts +10 -0
- package/build/commands/adula_runtime_health.js +27 -0
- package/build/commands/adula_uploads_prune.d.ts +10 -0
- package/build/commands/adula_uploads_prune.js +16 -0
- package/build/commands/app.d.ts +10 -0
- package/build/commands/app.js +64 -0
- package/build/commands/backup_create.d.ts +11 -0
- package/build/commands/backup_create.js +99 -0
- package/build/commands/backup_open.d.ts +16 -0
- package/build/commands/backup_open.js +48 -0
- package/build/commands/backup_restore_files.d.ts +11 -0
- package/build/commands/backup_restore_files.js +44 -0
- package/build/commands/backup_restore_test.d.ts +12 -0
- package/build/commands/backup_restore_test.js +188 -0
- package/build/commands/backup_seal.d.ts +12 -0
- package/build/commands/backup_seal.js +36 -0
- package/build/commands/backup_services.d.ts +33 -0
- package/build/commands/backup_services.js +6 -0
- package/build/commands/backup_verify.d.ts +9 -0
- package/build/commands/backup_verify.js +59 -0
- package/build/commands/backup_verify_snapshot.d.ts +10 -0
- package/build/commands/backup_verify_snapshot.js +37 -0
- package/build/commands/client.d.ts +16 -0
- package/build/commands/client.js +71 -0
- package/build/commands/doctor.js +21 -1
- package/build/commands/install.d.ts +8 -0
- package/build/commands/install.js +69 -45
- package/build/commands/main.d.ts +3 -2
- package/build/commands/main.js +32 -0
- package/build/commands/module_add.js +4 -20
- package/build/commands/resource.d.ts +2 -0
- package/build/commands/resource.js +12 -3
- package/build/commands/resource_migration.d.ts +15 -0
- package/build/commands/resource_migration.js +81 -0
- package/build/commands/resource_snapshot.d.ts +2 -0
- package/build/commands/resource_snapshot.js +21 -4
- package/build/commands/sla_evaluate.d.ts +14 -0
- package/build/commands/sla_evaluate.js +16 -0
- package/build/commands/ui.js +4 -2
- package/build/commands/ui_css.d.ts +9 -0
- package/build/commands/ui_css.js +19 -0
- package/build/commands/upgrade.d.ts +1 -1
- package/build/commands/upgrade.js +1 -1
- package/build/configure.js +1 -1
- package/build/database/migrations/1770000000020_two_factor.d.ts +5 -0
- package/build/database/migrations/1770000000020_two_factor.js +10 -0
- package/build/database/migrations/1770000000041_import_modes.d.ts +5 -0
- package/build/database/migrations/1770000000041_import_modes.js +10 -0
- package/build/database/migrations/1770000000042_sync_jobs.d.ts +5 -0
- package/build/database/migrations/1770000000042_sync_jobs.js +10 -0
- package/build/database/migrations/1770000000043_notification_channels.d.ts +5 -0
- package/build/database/migrations/1770000000043_notification_channels.js +10 -0
- package/build/database/migrations/1770000000044_channel_confirmations.d.ts +5 -0
- package/build/database/migrations/1770000000044_channel_confirmations.js +10 -0
- package/build/database/migrations/1770000000045_import_leases.d.ts +5 -0
- package/build/database/migrations/1770000000045_import_leases.js +10 -0
- package/build/database/migrations/1770000000046_workflow_delegation_sla.d.ts +5 -0
- package/build/database/migrations/1770000000046_workflow_delegation_sla.js +10 -0
- package/build/database/migrations/1770000000049_print_templates.d.ts +5 -0
- package/build/database/migrations/1770000000049_print_templates.js +10 -0
- package/build/database/migrations/1770000000051_comment_notifications.d.ts +5 -0
- package/build/database/migrations/1770000000051_comment_notifications.js +10 -0
- package/build/database/migrations/1770000000054_cloud_backups.d.ts +5 -0
- package/build/database/migrations/1770000000054_cloud_backups.js +10 -0
- package/build/database/migrations/1770000000055_ai_assistant.d.ts +5 -0
- package/build/database/migrations/1770000000055_ai_assistant.js +10 -0
- package/build/database/migrations/1770000000056_appearance.d.ts +5 -0
- package/build/database/migrations/1770000000056_appearance.js +10 -0
- package/build/index.d.ts +87 -11
- package/build/index.js +50 -7
- package/build/providers/adula_provider.d.ts +2 -0
- package/build/providers/adula_provider.js +7 -0
- package/build/src/admin/contracts.js +1 -1
- package/build/src/admin/controller.d.ts +14 -1
- package/build/src/admin/controller.js +91 -13
- package/build/src/admin/errors.d.ts +32 -0
- package/build/src/admin/errors.js +63 -1
- package/build/src/admin/presentation.d.ts +21 -2
- package/build/src/admin/resource/editor.d.ts +60 -0
- package/build/src/admin/resource/editor.js +345 -0
- package/build/src/admin/resource/export.d.ts +54 -0
- package/build/src/admin/resource/export.js +262 -0
- package/build/src/admin/resource/lifecycle.d.ts +23 -0
- package/build/src/admin/resource/lifecycle.js +91 -0
- package/build/src/admin/resource/query.d.ts +111 -0
- package/build/src/admin/resource/query.js +595 -0
- package/build/src/admin/resource/read.d.ts +109 -0
- package/build/src/admin/resource/read.js +388 -0
- package/build/src/admin/resource/shared.d.ts +93 -0
- package/build/src/admin/resource/shared.js +265 -0
- package/build/src/admin/resource/trash.d.ts +10 -0
- package/build/src/admin/resource/trash.js +56 -0
- package/build/src/admin/resource/write.d.ts +57 -0
- package/build/src/admin/resource/write.js +603 -0
- package/build/src/admin/resource_service.d.ts +80 -193
- package/build/src/admin/resource_service.js +81 -1343
- package/build/src/admin/resource_tools.d.ts +111 -0
- package/build/src/admin/resource_tools.js +139 -0
- package/build/src/ai/assistant.d.ts +100 -0
- package/build/src/ai/assistant.js +487 -0
- package/build/src/ai/client.d.ts +49 -0
- package/build/src/ai/client.js +85 -0
- package/build/src/ai/prompt.d.ts +23 -0
- package/build/src/ai/prompt.js +83 -0
- package/build/src/ai/settings.d.ts +56 -0
- package/build/src/ai/settings.js +214 -0
- package/build/src/ai/usage.d.ts +16 -0
- package/build/src/ai/usage.js +33 -0
- package/build/src/appearance/appearance.d.ts +134 -0
- package/build/src/appearance/appearance.js +461 -0
- package/build/src/appearance/colors.d.ts +43 -0
- package/build/src/appearance/colors.js +99 -0
- package/build/src/appearance/logo.d.ts +32 -0
- package/build/src/appearance/logo.js +112 -0
- package/build/src/attachments/attachment_service.d.ts +46 -2
- package/build/src/attachments/attachment_service.js +96 -4
- package/build/src/auth/actor_store.d.ts +5 -1
- package/build/src/auth/actor_store.js +8 -4
- package/build/src/auth/conditions.d.ts +5 -0
- package/build/src/auth/conditions.js +26 -3
- package/build/src/auth/sql.js +17 -5
- package/build/src/auth/two_factor.d.ts +96 -0
- package/build/src/auth/two_factor.js +259 -0
- package/build/src/backups/archive.d.ts +52 -0
- package/build/src/backups/archive.js +362 -0
- package/build/src/backups/cloud_backups.d.ts +133 -0
- package/build/src/backups/cloud_backups.js +650 -0
- package/build/src/backups/doctor.d.ts +7 -0
- package/build/src/backups/doctor.js +55 -0
- package/build/src/backups/providers.d.ts +81 -0
- package/build/src/backups/providers.js +470 -0
- package/build/src/collaboration/assignments.d.ts +59 -2
- package/build/src/collaboration/assignments.js +196 -74
- package/build/src/collaboration/record_collaboration.d.ts +21 -4
- package/build/src/collaboration/record_collaboration.js +126 -51
- package/build/src/collaboration/shared.d.ts +24 -0
- package/build/src/collaboration/shared.js +34 -0
- package/build/src/commands/app_scaffold.d.ts +72 -0
- package/build/src/commands/app_scaffold.js +285 -0
- package/build/src/commands/capabilities.js +35 -9
- package/build/src/commands/doctor.d.ts +24 -2
- package/build/src/commands/doctor.js +81 -6
- package/build/src/commands/gap_report.d.ts +1 -1
- package/build/src/commands/gap_report.js +2 -2
- package/build/src/commands/generator.d.ts +37 -1
- package/build/src/commands/generator.js +61 -18
- package/build/src/commands/snapshot.d.ts +24 -13
- package/build/src/commands/snapshot.js +54 -14
- package/build/src/commands/source_markers.d.ts +10 -0
- package/build/src/commands/source_markers.js +29 -0
- package/build/src/core/activity.d.ts +2 -0
- package/build/src/core/activity.js +7 -1
- package/build/src/core/administrators.d.ts +38 -0
- package/build/src/core/administrators.js +145 -0
- package/build/src/core/avatar_image.d.ts +33 -0
- package/build/src/core/avatar_image.js +278 -0
- package/build/src/core/html_sanitizer.d.ts +20 -0
- package/build/src/core/html_sanitizer.js +261 -0
- package/build/src/core/like.d.ts +5 -0
- package/build/src/core/like.js +5 -0
- package/build/src/core/lookups.d.ts +80 -0
- package/build/src/core/lookups.js +319 -0
- package/build/src/core/mail_settings.d.ts +83 -0
- package/build/src/core/mail_settings.js +274 -0
- package/build/src/core/mail_test.d.ts +16 -0
- package/build/src/core/mail_test.js +81 -6
- package/build/src/core/message_templates.d.ts +15 -2
- package/build/src/core/message_templates.js +81 -11
- package/build/src/core/module_seed.js +2 -1
- package/build/src/core/notification_channels.d.ts +235 -0
- package/build/src/core/notification_channels.js +660 -0
- package/build/src/core/roles.js +26 -9
- package/build/src/core/saved_views.d.ts +8 -1
- package/build/src/core/saved_views.js +91 -26
- package/build/src/core/sequences.d.ts +32 -0
- package/build/src/core/sequences.js +97 -0
- package/build/src/core/settings.d.ts +5 -0
- package/build/src/core/settings.js +31 -3
- package/build/src/core/template_language.d.ts +61 -0
- package/build/src/core/template_language.js +141 -0
- package/build/src/core/ui_preferences.d.ts +10 -0
- package/build/src/core/ui_preferences.js +15 -1
- package/build/src/core/user_avatars.d.ts +79 -0
- package/build/src/core/user_avatars.js +186 -0
- package/build/src/core/user_invitations.d.ts +5 -1
- package/build/src/core/user_invitations.js +7 -1
- package/build/src/core/user_preferences.d.ts +36 -0
- package/build/src/core/user_preferences.js +125 -0
- package/build/src/core/users.d.ts +2 -0
- package/build/src/core/users.js +5 -3
- package/build/src/database/calendar_days.d.ts +19 -0
- package/build/src/database/calendar_days.js +36 -0
- package/build/src/database/schema.d.ts +201 -2
- package/build/src/database/schema.js +713 -59
- package/build/src/database/schema_diff.d.ts +70 -0
- package/build/src/database/schema_diff.js +190 -0
- package/build/src/events/record_mutation.d.ts +2 -1
- package/build/src/events/record_mutation.js +1 -0
- package/build/src/integrations/api_client.d.ts +37 -0
- package/build/src/integrations/api_client.js +567 -0
- package/build/src/integrations/import_adapters.d.ts +37 -0
- package/build/src/integrations/import_adapters.js +35 -0
- package/build/src/integrations/imports.d.ts +130 -16
- package/build/src/integrations/imports.js +434 -73
- package/build/src/integrations/openapi.d.ts +3 -1
- package/build/src/integrations/openapi.js +465 -31
- package/build/src/integrations/outbound.d.ts +94 -0
- package/build/src/integrations/outbound.js +276 -0
- package/build/src/integrations/print.d.ts +22 -1
- package/build/src/integrations/print.js +68 -22
- package/build/src/integrations/print_templates.d.ts +216 -0
- package/build/src/integrations/print_templates.js +1017 -0
- package/build/src/integrations/sync_jobs.d.ts +139 -0
- package/build/src/integrations/sync_jobs.js +416 -0
- package/build/src/integrations/webhooks.d.ts +12 -12
- package/build/src/integrations/webhooks.js +34 -43
- package/build/src/integrations/xlsx.d.ts +41 -0
- package/build/src/integrations/xlsx.js +519 -0
- package/build/src/languages/ar.d.ts +11 -0
- package/build/src/languages/ar.js +358 -0
- package/build/src/languages/ar_ui.d.ts +461 -0
- package/build/src/languages/ar_ui.js +464 -0
- package/build/src/languages/catalog.d.ts +66 -0
- package/build/src/languages/catalog.js +152 -0
- package/build/src/languages/en.d.ts +7 -0
- package/build/src/languages/en.js +352 -0
- package/build/src/languages/en_ui.d.ts +3 -0
- package/build/src/languages/en_ui.js +458 -0
- package/build/src/languages/validation.d.ts +26 -0
- package/build/src/languages/validation.js +60 -0
- package/build/src/lists/bulk_actions.d.ts +89 -0
- package/build/src/lists/bulk_actions.js +268 -0
- package/build/src/lists/dashboards.d.ts +138 -0
- package/build/src/lists/dashboards.js +290 -0
- package/build/src/mcp/tools.js +9 -104
- package/build/src/resource/define_resource.d.ts +18 -1
- package/build/src/resource/define_resource.js +212 -18
- package/build/src/resource/definition.d.ts +44 -0
- package/build/src/resource/definition.js +171 -0
- package/build/src/resource/registry.js +15 -4
- package/build/src/resource/types.d.ts +313 -16
- package/build/src/resource/values.d.ts +56 -1
- package/build/src/resource/values.js +237 -5
- package/build/src/services/settings.d.ts +2 -0
- package/build/src/services/settings.js +5 -1
- package/build/src/upgrade/legacy.js +2 -2
- package/build/src/workflows/calendar.d.ts +9 -0
- package/build/src/workflows/calendar.js +44 -0
- package/build/src/workflows/define_workflow.d.ts +48 -3
- package/build/src/workflows/define_workflow.js +57 -13
- package/build/src/workflows/delegations.d.ts +107 -0
- package/build/src/workflows/delegations.js +326 -0
- package/build/src/workflows/engine.d.ts +80 -4
- package/build/src/workflows/engine.js +466 -86
- package/build/src/workflows/sla.d.ts +15 -0
- package/build/src/workflows/sla.js +8 -0
- package/build/stubs/resource_contract.txt +12 -1
- package/package.json +9 -2
|
@@ -1,17 +1,29 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
1
2
|
import Papa from 'papaparse';
|
|
2
3
|
import { subject } from '@casl/ability';
|
|
3
4
|
import { buildAbility } from '../auth/ability.js';
|
|
5
|
+
import { accessibleBy } from '../auth/sql.js';
|
|
6
|
+
import { columnName } from '../resource/define_resource.js';
|
|
4
7
|
import { AdulaError } from '../admin/errors.js';
|
|
5
8
|
import { notifyWithTemplate } from '../core/message_templates.js';
|
|
9
|
+
import { translate } from '../languages/catalog.js';
|
|
10
|
+
import { csvImportAdapter, extensionOf, importFile, } from './import_adapters.js';
|
|
11
|
+
import { XLSX_MIME_TYPE, xlsxImportAdapter } from './xlsx.js';
|
|
12
|
+
import { guardFormula } from '../admin/resource/export.js';
|
|
13
|
+
import writeXlsxFile from 'write-excel-file/node';
|
|
6
14
|
export const IMPORT_ROW_LIMIT = 5000;
|
|
7
15
|
const COLUMN_LIMIT = 100;
|
|
8
16
|
const ERROR_LIMIT = 200;
|
|
9
17
|
const FILE_LIMIT = 5 * 1024 * 1024;
|
|
10
|
-
|
|
18
|
+
// Lists (multiSelect, manyToMany) have no single-cell CSV form yet.
|
|
19
|
+
const UNSUPPORTED = new Set(['hasMany', 'attachment', 'json', 'multiSelect', 'manyToMany']);
|
|
20
|
+
const MODES = new Set(['create', 'upsert']);
|
|
11
21
|
const normalize = (value) => value
|
|
12
22
|
.trim()
|
|
13
23
|
.toLowerCase()
|
|
14
24
|
.replace(/[\s_-]+/g, '');
|
|
25
|
+
/** Template headers mark required columns with `*`; the mark never blocks matching. */
|
|
26
|
+
const headerKey = (value) => normalize(value.replace(/\s*\*+\s*$/, ''));
|
|
15
27
|
/** Converts one CSV cell to the field's stored representation, or throws a readable error. */
|
|
16
28
|
export function importCell(field, raw, lookups) {
|
|
17
29
|
const value = raw.trim();
|
|
@@ -62,6 +74,14 @@ export function importCell(field, raw, lookups) {
|
|
|
62
74
|
throw new Error('تاريخ ووقت غير صالحين');
|
|
63
75
|
return date.toISOString();
|
|
64
76
|
}
|
|
77
|
+
case 'decimal':
|
|
78
|
+
case 'percent': {
|
|
79
|
+
// "," and "٬" only as thousands grouping (1,234.5); "٫" is the Arabic decimal point.
|
|
80
|
+
const plain = latin.replace(/[\s%]/g, '').replace('٫', '.').replace(/٬/g, ',');
|
|
81
|
+
if (!/^-?(\d{1,3}(,\d{3})+|\d+)(\.\d+)?$/.test(plain))
|
|
82
|
+
throw new Error('رقم عشري غير صالح');
|
|
83
|
+
return plain.replace(/,/g, '');
|
|
84
|
+
}
|
|
65
85
|
case 'lookup': {
|
|
66
86
|
const key = lookups.get(normalize(value));
|
|
67
87
|
if (!key)
|
|
@@ -72,28 +92,62 @@ export function importCell(field, raw, lookups) {
|
|
|
72
92
|
return value;
|
|
73
93
|
}
|
|
74
94
|
}
|
|
95
|
+
/** How long a claimed batch stays reserved for one worker; renewed after every row. */
|
|
96
|
+
const LEASE_SECONDS = 300;
|
|
97
|
+
const leaseExpiry = (db) => db.raw(`now() + interval '${LEASE_SECONDS} seconds'`);
|
|
98
|
+
/** Raised inside a dry-run transaction so it always rolls back. */
|
|
99
|
+
class DryRunRollback extends Error {
|
|
100
|
+
}
|
|
75
101
|
/**
|
|
76
|
-
*
|
|
77
|
-
* to writable fields, and a worker saves each row through
|
|
78
|
-
* the importing user's current permissions. Row failures never stop
|
|
102
|
+
* File imports in batches: the upload is parsed by a format adapter (CSV built in) and
|
|
103
|
+
* stored, the user maps columns to writable fields, and a worker saves each row through
|
|
104
|
+
* ResourceService with the importing user's current permissions. Row failures never stop
|
|
105
|
+
* the batch. A dry run validates every row exactly like the real import inside a
|
|
106
|
+
* transaction that is rolled back, so nothing (records, activity, outbox events, sequence
|
|
107
|
+
* numbers, notifications) is kept; the batch then waits as 'validated' for the real run.
|
|
108
|
+
* Upsert mode updates the live record matched by a unique field and creates the rest.
|
|
79
109
|
*/
|
|
80
110
|
export class ImportBatches {
|
|
81
111
|
db;
|
|
82
112
|
registry;
|
|
83
113
|
resources;
|
|
84
114
|
actors;
|
|
85
|
-
|
|
115
|
+
formats = [csvImportAdapter, xlsxImportAdapter];
|
|
116
|
+
constructor(db, registry, resources, actors, options = {}) {
|
|
86
117
|
this.db = db;
|
|
87
118
|
this.registry = registry;
|
|
88
119
|
this.resources = resources;
|
|
89
120
|
this.actors = actors;
|
|
121
|
+
for (const adapter of options.adapters ?? [])
|
|
122
|
+
this.registerAdapter(adapter);
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Adds a file format, for example an ODS adapter. A later adapter with the same
|
|
126
|
+
* name replaces the earlier one; CSV stays the fallback for text without an extension.
|
|
127
|
+
*/
|
|
128
|
+
registerAdapter(adapter) {
|
|
129
|
+
if (!/^[a-z0-9][a-z0-9_-]*$/.test(adapter.name) || !adapter.extensions.length)
|
|
130
|
+
throw new Error('An import adapter needs a lower-case name and at least one extension');
|
|
131
|
+
this.formats = [...this.formats.filter((item) => item.name !== adapter.name), adapter];
|
|
132
|
+
return this;
|
|
133
|
+
}
|
|
134
|
+
/** Registered formats, CSV first. */
|
|
135
|
+
adapters() {
|
|
136
|
+
return this.formats;
|
|
90
137
|
}
|
|
91
|
-
/**
|
|
92
|
-
|
|
138
|
+
/** Every accepted upload extension, for the upload validator and the file picker. */
|
|
139
|
+
extensions() {
|
|
140
|
+
return [...new Set(this.formats.flatMap((adapter) => adapter.extensions))];
|
|
141
|
+
}
|
|
142
|
+
/** Writable, importable fields for this actor; upsert also needs update permission. */
|
|
143
|
+
targets(name, actor, mode = 'create') {
|
|
93
144
|
const resource = this.registry.get(name);
|
|
94
145
|
const ability = buildAbility(actor.rules, this.registry.all());
|
|
95
146
|
if (!resource.actions.includes('create') || !ability.can('create', name))
|
|
96
147
|
throw new AdulaError(403, 'E_FORBIDDEN', 'ليس لديك صلاحية الإضافة إلى هذا الكيان');
|
|
148
|
+
const upsert = mode === 'upsert';
|
|
149
|
+
if (upsert && (!resource.actions.includes('update') || !ability.can('update', name)))
|
|
150
|
+
throw new AdulaError(403, 'E_FORBIDDEN', 'الإضافة أو التحديث يتطلب صلاحية التعديل أيضاً');
|
|
97
151
|
const fields = resource.form
|
|
98
152
|
.filter((key) => {
|
|
99
153
|
const field = resource.fields[key];
|
|
@@ -101,13 +155,15 @@ export class ImportBatches {
|
|
|
101
155
|
!field.sequence &&
|
|
102
156
|
actor.permissionLevel >=
|
|
103
157
|
Math.max(field.permissionLevel ?? 0, resource.hidden?.includes(key) ? 1 : 0) &&
|
|
104
|
-
ability.can('create', subject(name, {}), key)
|
|
158
|
+
ability.can('create', subject(name, {}), key) &&
|
|
159
|
+
(!upsert || ability.can('update', subject(name, {}), key)));
|
|
105
160
|
})
|
|
106
161
|
.map((key) => ({
|
|
107
162
|
key,
|
|
108
163
|
label: resource.fields[key].label.ar,
|
|
109
164
|
type: resource.fields[key].type,
|
|
110
165
|
required: Boolean(resource.fields[key].required),
|
|
166
|
+
unique: Boolean(resource.fields[key].unique),
|
|
111
167
|
}));
|
|
112
168
|
// An inherited scope takes the unit from the parent row instead (#45).
|
|
113
169
|
if (resource.scoped && !resource.scope)
|
|
@@ -116,29 +172,72 @@ export class ImportBatches {
|
|
|
116
172
|
label: 'الوحدة التنظيمية (رقم)',
|
|
117
173
|
type: 'integer',
|
|
118
174
|
required: true,
|
|
175
|
+
unique: false,
|
|
119
176
|
});
|
|
120
177
|
return fields;
|
|
121
178
|
}
|
|
179
|
+
/**
|
|
180
|
+
* A CSV header row of the actor's importable fields in Arabic, required ones marked
|
|
181
|
+
* with `*`, with a UTF-8 byte-order mark so spreadsheet programs show Arabic. The
|
|
182
|
+
* headers map automatically when the filled template is uploaded.
|
|
183
|
+
*/
|
|
184
|
+
template(name, actor, mode = 'create') {
|
|
185
|
+
const targets = this.targets(name, actor, mode);
|
|
186
|
+
const headers = targets.map((target) => `${target.label}${target.required ? ' *' : ''}`);
|
|
187
|
+
return {
|
|
188
|
+
fileName: `${name}-template.csv`,
|
|
189
|
+
content: `\uFEFF${Papa.unparse([headers])}\n`,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* The same template as an Excel workbook: one right-to-left sheet whose bold header row
|
|
194
|
+
* holds the actor's importable fields, required ones marked with `*`.
|
|
195
|
+
*/
|
|
196
|
+
async xlsxTemplate(name, actor, mode = 'create') {
|
|
197
|
+
const targets = this.targets(name, actor, mode);
|
|
198
|
+
const header = targets.map((target) => ({
|
|
199
|
+
value: guardFormula(`${target.label}${target.required ? ' *' : ''}`),
|
|
200
|
+
fontWeight: 'bold',
|
|
201
|
+
}));
|
|
202
|
+
const content = await writeXlsxFile([header], {
|
|
203
|
+
rightToLeft: true,
|
|
204
|
+
stickyRowsCount: 1,
|
|
205
|
+
}).toBuffer();
|
|
206
|
+
return {
|
|
207
|
+
fileName: `${name}-template.xlsx`,
|
|
208
|
+
mimeType: XLSX_MIME_TYPE,
|
|
209
|
+
content: new Uint8Array(content.buffer, content.byteOffset, content.byteLength),
|
|
210
|
+
};
|
|
211
|
+
}
|
|
122
212
|
async create(name, actor, input) {
|
|
123
|
-
const
|
|
213
|
+
const mode = this.mode(input.mode ?? 'create');
|
|
214
|
+
const targets = this.targets(name, actor, mode);
|
|
215
|
+
const matchField = input.matchField ? this.matchField(targets, input.matchField) : null;
|
|
124
216
|
const fileName = typeof input.fileName === 'string' ? input.fileName.slice(0, 200) : 'import.csv';
|
|
125
|
-
|
|
217
|
+
let bytes;
|
|
218
|
+
if (typeof input.content === 'string') {
|
|
219
|
+
if (!input.content.trim())
|
|
220
|
+
throw new AdulaError(422, 'E_IMPORT_FILE', 'الملف فارغ');
|
|
221
|
+
bytes = Buffer.from(input.content, input.encoding === 'base64' ? 'base64' : 'utf8');
|
|
222
|
+
}
|
|
223
|
+
else if (input.content instanceof Uint8Array)
|
|
224
|
+
bytes = input.content;
|
|
225
|
+
else
|
|
126
226
|
throw new AdulaError(422, 'E_IMPORT_FILE', 'الملف فارغ');
|
|
127
|
-
if (
|
|
227
|
+
if (!bytes.byteLength)
|
|
228
|
+
throw new AdulaError(422, 'E_IMPORT_FILE', 'الملف فارغ');
|
|
229
|
+
if (bytes.byteLength > FILE_LIMIT)
|
|
128
230
|
throw new AdulaError(422, 'E_IMPORT_FILE', 'حجم الملف يتجاوز 5 ميغابايت');
|
|
129
|
-
const
|
|
130
|
-
|
|
131
|
-
});
|
|
132
|
-
if (parsed.errors.some((error) => error.type !== 'Delimiter'))
|
|
133
|
-
throw new AdulaError(422, 'E_IMPORT_FILE', `تعذر قراءة CSV: ${parsed.errors[0].message}`);
|
|
134
|
-
const [headers = [], ...rows] = parsed.data;
|
|
231
|
+
const file = importFile(fileName, bytes, typeof input.mimeType === 'string' ? input.mimeType : undefined);
|
|
232
|
+
const adapter = this.adapterFor(file, input.format, typeof input.content === 'string');
|
|
233
|
+
const { headers, rows } = await adapter.parse(file);
|
|
135
234
|
if (!headers.length || headers.length > COLUMN_LIMIT)
|
|
136
235
|
throw new AdulaError(422, 'E_IMPORT_FILE', 'يجب أن يحتوي الملف صف عناوين (100 عمود كحد أقصى)');
|
|
137
236
|
if (!rows.length || rows.length > IMPORT_ROW_LIMIT)
|
|
138
237
|
throw new AdulaError(422, 'E_IMPORT_ROWS', `عدد الصفوف بين 1 و${IMPORT_ROW_LIMIT}`);
|
|
139
238
|
const mapping = {};
|
|
140
239
|
headers.forEach((header, index) => {
|
|
141
|
-
const target = targets.find((field) => [field.key, field.label].some((alias) => normalize(alias) ===
|
|
240
|
+
const target = targets.find((field) => [field.key, field.label].some((alias) => normalize(alias) === headerKey(String(header))));
|
|
142
241
|
if (target && !Object.values(mapping).includes(target.key))
|
|
143
242
|
mapping[String(index)] = target.key;
|
|
144
243
|
});
|
|
@@ -148,9 +247,12 @@ export class ImportBatches {
|
|
|
148
247
|
user_id: actor.id,
|
|
149
248
|
file_name: fileName,
|
|
150
249
|
status: 'mapping',
|
|
151
|
-
|
|
250
|
+
mode,
|
|
251
|
+
match_field: matchField,
|
|
252
|
+
dry_run: input.dryRun === true,
|
|
253
|
+
headers: JSON.stringify(headers.map(String)),
|
|
152
254
|
mapping: JSON.stringify(mapping),
|
|
153
|
-
rows: JSON.stringify(rows.map((cells) => cells.slice(0, headers.length))),
|
|
255
|
+
rows: JSON.stringify(rows.map((cells) => cells.slice(0, headers.length).map((cell) => String(cell ?? '')))),
|
|
154
256
|
total: rows.length,
|
|
155
257
|
})
|
|
156
258
|
.returning('*');
|
|
@@ -158,22 +260,27 @@ export class ImportBatches {
|
|
|
158
260
|
}
|
|
159
261
|
async show(id, actor) {
|
|
160
262
|
const row = await this.owned(id, actor);
|
|
161
|
-
return this.present(row, this.safeTargets(row.resource, actor));
|
|
263
|
+
return this.present(row, this.safeTargets(row.resource, actor, row.mode));
|
|
162
264
|
}
|
|
163
265
|
async list(actor) {
|
|
164
266
|
const rows = await this.db('import_batches')
|
|
165
267
|
.where('user_id', actor.id)
|
|
166
268
|
.orderBy('id', 'desc')
|
|
167
269
|
.limit(50)
|
|
168
|
-
.select('id', 'resource', 'file_name', 'status', 'headers', 'mapping', 'total', 'processed', 'created', 'failed', 'errors', 'created_at', 'finished_at');
|
|
169
|
-
return rows.map((row) => this.present({ ...row, rows: [] }, []));
|
|
270
|
+
.select('id', 'resource', 'file_name', 'status', 'mode', 'match_field', 'dry_run', 'headers', 'mapping', 'total', 'processed', 'created', 'updated', 'failed', 'errors', 'created_at', 'finished_at');
|
|
271
|
+
return rows.map((row) => this.present({ ...row, rows: [], report: [] }, []));
|
|
170
272
|
}
|
|
171
|
-
/**
|
|
172
|
-
|
|
273
|
+
/**
|
|
274
|
+
* Stores the column mapping and queues the batch for the worker. A batch in 'mapping'
|
|
275
|
+
* runs as created (dry run or not) unless `dryRun` says otherwise; a 'validated' batch
|
|
276
|
+
* runs for real by default, so the same batch is imported after its dry run.
|
|
277
|
+
*/
|
|
278
|
+
async start(id, actor, mapping, options = {}) {
|
|
173
279
|
const row = await this.owned(id, actor);
|
|
174
|
-
if (row.status
|
|
280
|
+
if (!['mapping', 'validated'].includes(row.status))
|
|
175
281
|
throw new AdulaError(409, 'E_IMPORT_STATE', 'بدأت معالجة هذه الدفعة بالفعل');
|
|
176
|
-
const
|
|
282
|
+
const mode = this.mode(options.mode ?? row.mode ?? 'create');
|
|
283
|
+
const targets = this.targets(row.resource, actor, mode);
|
|
177
284
|
if (!mapping || typeof mapping !== 'object' || Array.isArray(mapping))
|
|
178
285
|
throw new AdulaError(422, 'E_IMPORT_MAPPING', 'مطابقة الأعمدة غير صالحة');
|
|
179
286
|
const clean = {};
|
|
@@ -184,6 +291,10 @@ export class ImportBatches {
|
|
|
184
291
|
const index = Number(column);
|
|
185
292
|
if (!Number.isInteger(index) || index < 0 || index >= row.headers.length)
|
|
186
293
|
throw new AdulaError(422, 'E_IMPORT_MAPPING', 'عمود غير موجود');
|
|
294
|
+
// Files cannot come from a spreadsheet cell: they are uploaded on the record page.
|
|
295
|
+
const named = this.registry.get(row.resource).fields;
|
|
296
|
+
if (typeof key === 'string' && Object.hasOwn(named, key) && named[key].type === 'attachment')
|
|
297
|
+
throw new AdulaError(422, 'E_IMPORT_MAPPING', `لا يمكن استيراد المرفقات من ملف؛ ارفعها من صفحة السجل: ${named[key].label.ar}`);
|
|
187
298
|
if (typeof key !== 'string' || !targets.some((target) => target.key === key) || used.has(key))
|
|
188
299
|
throw new AdulaError(422, 'E_IMPORT_MAPPING', `حقل غير قابل للاستيراد أو مكرر: ${String(key)}`);
|
|
189
300
|
used.add(key);
|
|
@@ -192,19 +303,60 @@ export class ImportBatches {
|
|
|
192
303
|
const missing = targets.filter((target) => target.required && !used.has(target.key));
|
|
193
304
|
if (missing.length)
|
|
194
305
|
throw new AdulaError(422, 'E_IMPORT_MAPPING', `حقول مطلوبة بلا عمود: ${missing.map((target) => target.label).join('، ')}`);
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
306
|
+
let matchField = null;
|
|
307
|
+
if (mode === 'upsert') {
|
|
308
|
+
matchField = this.matchField(targets, options.matchField ?? row.match_field);
|
|
309
|
+
if (!used.has(matchField))
|
|
310
|
+
throw new AdulaError(422, 'E_IMPORT_MATCH', 'اربط حقل المطابقة بعمود من الملف');
|
|
311
|
+
}
|
|
312
|
+
const dryRun = typeof options.dryRun === 'boolean'
|
|
313
|
+
? options.dryRun
|
|
314
|
+
: row.status === 'validated'
|
|
315
|
+
? false
|
|
316
|
+
: Boolean(row.dry_run);
|
|
317
|
+
// Conditional on the state read above: of two concurrent starts only one queues the
|
|
318
|
+
// batch, and a batch a worker already claimed is never reset to row 0.
|
|
319
|
+
const queued = await this.db('import_batches')
|
|
320
|
+
.where({ id: row.id, status: row.status })
|
|
321
|
+
.whereIn('status', ['mapping', 'validated'])
|
|
322
|
+
.update({
|
|
323
|
+
mapping: JSON.stringify(clean),
|
|
324
|
+
mode,
|
|
325
|
+
match_field: matchField,
|
|
326
|
+
dry_run: dryRun,
|
|
327
|
+
status: 'queued',
|
|
328
|
+
processed: 0,
|
|
329
|
+
created: 0,
|
|
330
|
+
updated: 0,
|
|
331
|
+
failed: 0,
|
|
332
|
+
errors: JSON.stringify([]),
|
|
333
|
+
report: JSON.stringify([]),
|
|
334
|
+
started_at: null,
|
|
335
|
+
finished_at: null,
|
|
336
|
+
lease_id: null,
|
|
337
|
+
lease_expires_at: null,
|
|
338
|
+
});
|
|
339
|
+
if (!queued)
|
|
340
|
+
throw new AdulaError(409, 'E_IMPORT_STATE', 'بدأت معالجة هذه الدفعة بالفعل');
|
|
198
341
|
return this.show(row.id, actor);
|
|
199
342
|
}
|
|
200
343
|
/**
|
|
201
|
-
* Worker step: claims one queued or running batch and imports
|
|
202
|
-
*
|
|
344
|
+
* Worker step: claims one queued or running batch and imports (or, for a dry run,
|
|
345
|
+
* validates) up to `chunk` rows. The claim is a lease (lease_id, lease_expires_at, renewed
|
|
346
|
+
* after every row): concurrent workers never share a batch, and an expired lease is taken
|
|
347
|
+
* over, resuming after the last committed row. Import progress is committed per row in
|
|
348
|
+
* the row's own transaction; a dry run records its progress and report once per chunk.
|
|
349
|
+
* Returns null when no batch is free, or when this worker's lease was taken over (it then
|
|
350
|
+
* writes nothing further).
|
|
203
351
|
*/
|
|
204
352
|
async process(chunk = 200) {
|
|
353
|
+
const lease = randomUUID();
|
|
205
354
|
const batch = await this.db.transaction(async (trx) => {
|
|
355
|
+
// A batch is free when no worker holds an unexpired lease on it; an expired lease
|
|
356
|
+
// (a crashed or stalled worker) is taken over and processing resumes after its last row.
|
|
206
357
|
const row = await trx('import_batches')
|
|
207
358
|
.whereIn('status', ['queued', 'running'])
|
|
359
|
+
.where((free) => free.whereNull('lease_id').orWhere('lease_expires_at', '<', trx.fn.now()))
|
|
208
360
|
.orderBy('id')
|
|
209
361
|
.forUpdate()
|
|
210
362
|
.skipLocked()
|
|
@@ -213,11 +365,19 @@ export class ImportBatches {
|
|
|
213
365
|
return null;
|
|
214
366
|
await trx('import_batches')
|
|
215
367
|
.where('id', row.id)
|
|
216
|
-
.update({
|
|
368
|
+
.update({
|
|
369
|
+
status: 'running',
|
|
370
|
+
started_at: row.started_at ?? trx.fn.now(),
|
|
371
|
+
lease_id: lease,
|
|
372
|
+
lease_expires_at: leaseExpiry(trx),
|
|
373
|
+
});
|
|
217
374
|
return row;
|
|
218
375
|
});
|
|
219
376
|
if (!batch)
|
|
220
377
|
return null;
|
|
378
|
+
// Every later write names this lease; one that affects no row means another worker
|
|
379
|
+
// took the batch over, and this worker stops without writing anything else.
|
|
380
|
+
const owned = (db) => db('import_batches').where({ id: batch.id, lease_id: lease });
|
|
221
381
|
const actor = await this.actors.load(Number(batch.user_id));
|
|
222
382
|
const resource = this.registry.get(batch.resource);
|
|
223
383
|
const lookupGroups = new Map();
|
|
@@ -235,62 +395,247 @@ export class ImportBatches {
|
|
|
235
395
|
}
|
|
236
396
|
const mapping = batch.mapping;
|
|
237
397
|
const rows = batch.rows;
|
|
238
|
-
|
|
239
|
-
const
|
|
398
|
+
const dryRun = Boolean(batch.dry_run);
|
|
399
|
+
const mode = batch.mode === 'upsert' ? 'upsert' : 'create';
|
|
400
|
+
const matchField = mode === 'upsert' ? batch.match_field : null;
|
|
401
|
+
// A dry run rolls each row back, so it finds duplicates within the file itself.
|
|
402
|
+
const earlier = dryRun ? this.earlierDuplicates(resource, mapping, rows, matchField) : null;
|
|
403
|
+
let { processed, created, updated, failed } = batch;
|
|
404
|
+
let errors = [...batch.errors];
|
|
405
|
+
const report = [...(batch.report ?? [])];
|
|
240
406
|
const end = Math.min(rows.length, processed + chunk);
|
|
241
407
|
for (let index = processed; index < end; index++) {
|
|
242
408
|
const cells = rows[index];
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
409
|
+
// One row: convert, then save (in a savepoint of `trx` for a real import).
|
|
410
|
+
const importRow = async (trx) => {
|
|
411
|
+
const input = {};
|
|
412
|
+
try {
|
|
413
|
+
for (const [column, key] of Object.entries(mapping)) {
|
|
414
|
+
const raw = cells[Number(column)] ?? '';
|
|
415
|
+
if (key === 'orgUnitId') {
|
|
416
|
+
input.orgUnitId = raw.trim() === '' ? null : Number(raw.trim());
|
|
417
|
+
continue;
|
|
418
|
+
}
|
|
419
|
+
try {
|
|
420
|
+
input[key] = importCell(resource.fields[key], raw, lookupGroups.get(key) ?? new Map());
|
|
421
|
+
}
|
|
422
|
+
catch (error) {
|
|
423
|
+
throw new Error(`${resource.fields[key].label.ar}: ${error.message}`);
|
|
424
|
+
}
|
|
256
425
|
}
|
|
426
|
+
const duplicate = earlier?.duplicates.get(index);
|
|
427
|
+
if (duplicate)
|
|
428
|
+
throw new Error(duplicate);
|
|
429
|
+
let action = await this.saveRow(resource, actor, input, matchField, dryRun, trx);
|
|
430
|
+
// A repeated match value updates the record the earlier row of the file created.
|
|
431
|
+
if (earlier?.repeats.has(index))
|
|
432
|
+
action = 'update';
|
|
433
|
+
return { row: index + 2, ok: true, action };
|
|
434
|
+
}
|
|
435
|
+
catch (error) {
|
|
436
|
+
return { row: index + 2, ok: false, message: rowMessage(error) };
|
|
257
437
|
}
|
|
258
|
-
|
|
259
|
-
|
|
438
|
+
};
|
|
439
|
+
const count = (outcome) => {
|
|
440
|
+
const next = { created, updated, failed, errors };
|
|
441
|
+
if (!outcome.ok) {
|
|
442
|
+
next.failed++;
|
|
443
|
+
if (errors.length < ERROR_LIMIT)
|
|
444
|
+
next.errors = [...errors, { row: outcome.row, message: outcome.message ?? '' }];
|
|
445
|
+
}
|
|
446
|
+
else if (outcome.action === 'create')
|
|
447
|
+
next.created++;
|
|
448
|
+
else
|
|
449
|
+
next.updated++;
|
|
450
|
+
return next;
|
|
451
|
+
};
|
|
452
|
+
let next;
|
|
453
|
+
if (dryRun) {
|
|
454
|
+
const outcome = await importRow();
|
|
455
|
+
next = count(outcome);
|
|
456
|
+
report.push(outcome);
|
|
457
|
+
// A dry run saves nothing: only the lease is renewed per row; progress and the
|
|
458
|
+
// report are written once per chunk, and a crash only validates that chunk again.
|
|
459
|
+
if (!(await owned(this.db).update({ lease_expires_at: leaseExpiry(this.db) })))
|
|
460
|
+
return null;
|
|
260
461
|
}
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
462
|
+
else {
|
|
463
|
+
// A real import saves the row and its progress in one transaction that first locks
|
|
464
|
+
// the batch under this lease at this row, so every row is imported exactly once
|
|
465
|
+
// even when an expired lease was taken over by another worker meanwhile.
|
|
466
|
+
next = await this.db.transaction(async (trx) => {
|
|
467
|
+
if (!(await owned(trx).where('processed', index).forUpdate().first('id')))
|
|
468
|
+
return null;
|
|
469
|
+
const counts = count(await importRow(trx));
|
|
470
|
+
await owned(trx).update({
|
|
471
|
+
processed: index + 1,
|
|
472
|
+
created: counts.created,
|
|
473
|
+
updated: counts.updated,
|
|
474
|
+
failed: counts.failed,
|
|
475
|
+
errors: JSON.stringify(counts.errors),
|
|
476
|
+
lease_expires_at: leaseExpiry(trx),
|
|
269
477
|
});
|
|
478
|
+
return counts;
|
|
479
|
+
});
|
|
480
|
+
if (!next)
|
|
481
|
+
return null;
|
|
270
482
|
}
|
|
483
|
+
;
|
|
484
|
+
({ created, updated, failed, errors } = next);
|
|
271
485
|
processed = index + 1;
|
|
272
|
-
await this.db('import_batches')
|
|
273
|
-
.where('id', batch.id)
|
|
274
|
-
.update({ processed, created, failed, errors: JSON.stringify(errors) });
|
|
275
486
|
}
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
487
|
+
const finished = processed >= rows.length;
|
|
488
|
+
if (dryRun) {
|
|
489
|
+
const written = await owned(this.db).update({
|
|
490
|
+
processed,
|
|
491
|
+
created,
|
|
492
|
+
updated,
|
|
493
|
+
failed,
|
|
494
|
+
errors: JSON.stringify(errors),
|
|
495
|
+
report: JSON.stringify(report),
|
|
496
|
+
// The rows stay for the real import; a dry run sends no notification.
|
|
497
|
+
...(finished ? { status: 'validated', finished_at: this.db.fn.now() } : {}),
|
|
498
|
+
lease_id: null,
|
|
499
|
+
lease_expires_at: null,
|
|
500
|
+
});
|
|
501
|
+
if (!written)
|
|
502
|
+
return null;
|
|
503
|
+
}
|
|
504
|
+
else if (finished) {
|
|
505
|
+
const written = await this.db.transaction(async (trx) => {
|
|
506
|
+
const count = await owned(trx)
|
|
280
507
|
// Row data is only needed while importing; the log keeps counts and errors.
|
|
281
|
-
.update({
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
508
|
+
.update({
|
|
509
|
+
status: 'done',
|
|
510
|
+
finished_at: trx.fn.now(),
|
|
511
|
+
rows: JSON.stringify([]),
|
|
512
|
+
lease_id: null,
|
|
513
|
+
lease_expires_at: null,
|
|
286
514
|
});
|
|
515
|
+
if (count)
|
|
516
|
+
await notifyWithTemplate(trx, Number(batch.user_id), 'import.finished', {
|
|
517
|
+
resource: resource.label.ar,
|
|
518
|
+
created: created + updated,
|
|
519
|
+
failed,
|
|
520
|
+
});
|
|
521
|
+
return count;
|
|
522
|
+
});
|
|
523
|
+
if (!written)
|
|
524
|
+
return null;
|
|
525
|
+
}
|
|
526
|
+
else if (!(await owned(this.db).update({ lease_id: null, lease_expires_at: null })))
|
|
527
|
+
// The chunk is done; release the batch so the next tick (any worker) continues it.
|
|
528
|
+
return null;
|
|
529
|
+
return { id: Number(batch.id), processed, created, updated, failed };
|
|
530
|
+
}
|
|
531
|
+
/**
|
|
532
|
+
* Saves one row through ResourceService. Upsert first looks for the live record with
|
|
533
|
+
* the same match value among the records the actor may update (scope and role rules as
|
|
534
|
+
* SQL, as list() applies them); ResourceService then authorizes the update and checks
|
|
535
|
+
* the record's current `version`. A value held only by a record outside that reach takes
|
|
536
|
+
* the create path and answers exactly as a create of it would (a duplicate from the
|
|
537
|
+
* unique index), so the row never tells the importer where the value is stored. A real import runs in a
|
|
538
|
+
* savepoint of `trx` (the caller's progress transaction); a dry run does the same inside
|
|
539
|
+
* a transaction that is always rolled back.
|
|
540
|
+
*/
|
|
541
|
+
async saveRow(resource, actor, input, matchField, dryRun, trx) {
|
|
542
|
+
const work = async (scope) => {
|
|
543
|
+
let id;
|
|
544
|
+
if (matchField) {
|
|
545
|
+
const field = resource.fields[matchField];
|
|
546
|
+
const value = input[matchField];
|
|
547
|
+
if (value === null || value === undefined)
|
|
548
|
+
throw new Error(`${field.label.ar}: قيمة المطابقة مطلوبة`);
|
|
549
|
+
const reachable = scope(`${resource.name} as r`)
|
|
550
|
+
.whereNull('r.deleted_at')
|
|
551
|
+
.where(`r.${field.column ?? columnName(matchField)}`, value);
|
|
552
|
+
accessibleBy(reachable, buildAbility(actor.rules, this.registry.all()), actor, 'update', resource);
|
|
553
|
+
const found = await reachable.first(['r.id', ...(resource.version ? ['r.version'] : [])]);
|
|
554
|
+
if (found) {
|
|
555
|
+
id = Number(found.id);
|
|
556
|
+
if (resource.version)
|
|
557
|
+
input.version = Number(found.version);
|
|
558
|
+
// An empty cell keeps the stored value of a matched record instead of clearing it.
|
|
559
|
+
for (const key of Object.keys(input))
|
|
560
|
+
if (input[key] === null)
|
|
561
|
+
delete input[key];
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
await this.resources.save(resource.name, actor, input, id, scope);
|
|
565
|
+
return id === undefined ? 'create' : 'update';
|
|
566
|
+
};
|
|
567
|
+
if (!dryRun)
|
|
568
|
+
return (trx ?? this.db).transaction(work);
|
|
569
|
+
let action = 'create';
|
|
570
|
+
try {
|
|
571
|
+
await this.db.transaction(async (scope) => {
|
|
572
|
+
action = await work(scope);
|
|
573
|
+
throw new DryRunRollback();
|
|
287
574
|
});
|
|
288
575
|
}
|
|
289
|
-
|
|
576
|
+
catch (error) {
|
|
577
|
+
if (!(error instanceof DryRunRollback))
|
|
578
|
+
throw error;
|
|
579
|
+
}
|
|
580
|
+
return action;
|
|
290
581
|
}
|
|
291
|
-
|
|
582
|
+
/**
|
|
583
|
+
* Rows whose unique value repeats an earlier row of the same file. In upsert mode a
|
|
584
|
+
* repeated match value is not an error: that row would update the earlier row's record.
|
|
585
|
+
*/
|
|
586
|
+
earlierDuplicates(resource, mapping, rows, matchField) {
|
|
587
|
+
const duplicates = new Map();
|
|
588
|
+
const repeats = new Set();
|
|
589
|
+
for (const [column, key] of Object.entries(mapping)) {
|
|
590
|
+
if (!resource.fields[key]?.unique)
|
|
591
|
+
continue;
|
|
592
|
+
const seen = new Map();
|
|
593
|
+
rows.forEach((cells, index) => {
|
|
594
|
+
const value = (cells[Number(column)] ?? '').trim();
|
|
595
|
+
if (!value)
|
|
596
|
+
return;
|
|
597
|
+
const first = seen.get(value);
|
|
598
|
+
if (first === undefined)
|
|
599
|
+
seen.set(value, index);
|
|
600
|
+
else if (key === matchField)
|
|
601
|
+
repeats.add(index);
|
|
602
|
+
else if (!duplicates.has(index))
|
|
603
|
+
duplicates.set(index, `${resource.fields[key].label.ar}: القيمة مكررة في الملف (الصف ${first + 2})`);
|
|
604
|
+
});
|
|
605
|
+
}
|
|
606
|
+
return { duplicates, repeats };
|
|
607
|
+
}
|
|
608
|
+
adapterFor(file, format, text) {
|
|
609
|
+
if (format !== undefined && format !== null && format !== '') {
|
|
610
|
+
const named = this.formats.find((adapter) => adapter.name === format);
|
|
611
|
+
if (!named)
|
|
612
|
+
throw new AdulaError(422, 'E_IMPORT_FORMAT', 'صيغة الملف غير مدعومة');
|
|
613
|
+
return named;
|
|
614
|
+
}
|
|
615
|
+
const byExtension = this.formats.filter((adapter) => adapter.extensions.includes(extensionOf(file.fileName)));
|
|
616
|
+
const detected = [...byExtension].reverse().find((adapter) => adapter.detect(file)) ??
|
|
617
|
+
[...this.formats].reverse().find((adapter) => adapter.detect(file));
|
|
618
|
+
if (detected)
|
|
619
|
+
return detected;
|
|
620
|
+
// Text without a recognizable name keeps the historical CSV behaviour.
|
|
621
|
+
if (text && !extensionOf(file.fileName))
|
|
622
|
+
return csvImportAdapter;
|
|
623
|
+
throw new AdulaError(422, 'E_IMPORT_FORMAT', 'صيغة الملف غير مدعومة');
|
|
624
|
+
}
|
|
625
|
+
mode(value) {
|
|
626
|
+
if (typeof value !== 'string' || !MODES.has(value))
|
|
627
|
+
throw new AdulaError(422, 'E_IMPORT_MODE', 'نمط الاستيراد غير صالح');
|
|
628
|
+
return value;
|
|
629
|
+
}
|
|
630
|
+
matchField(targets, value) {
|
|
631
|
+
const target = targets.find((item) => item.key === value);
|
|
632
|
+
if (!target || !target.unique)
|
|
633
|
+
throw new AdulaError(422, 'E_IMPORT_MATCH', 'حقل المطابقة يجب أن يكون حقلاً فريداً قابلاً للاستيراد');
|
|
634
|
+
return target.key;
|
|
635
|
+
}
|
|
636
|
+
safeTargets(name, actor, mode) {
|
|
292
637
|
try {
|
|
293
|
-
return this.targets(name, actor);
|
|
638
|
+
return this.targets(name, actor, mode === 'upsert' ? 'upsert' : 'create');
|
|
294
639
|
}
|
|
295
640
|
catch {
|
|
296
641
|
return [];
|
|
@@ -317,17 +662,33 @@ export class ImportBatches {
|
|
|
317
662
|
resourceLabel: label,
|
|
318
663
|
fileName: String(row.file_name),
|
|
319
664
|
status: row.status,
|
|
665
|
+
mode: row.mode === 'upsert' ? 'upsert' : 'create',
|
|
666
|
+
matchField: row.match_field ?? null,
|
|
667
|
+
dryRun: Boolean(row.dry_run),
|
|
320
668
|
headers: row.headers ?? [],
|
|
321
669
|
mapping: row.mapping ?? {},
|
|
322
670
|
sample: (row.rows ?? []).slice(0, 5),
|
|
323
671
|
total: Number(row.total),
|
|
324
672
|
processed: Number(row.processed ?? 0),
|
|
325
673
|
created: Number(row.created ?? 0),
|
|
674
|
+
updated: Number(row.updated ?? 0),
|
|
326
675
|
failed: Number(row.failed ?? 0),
|
|
327
676
|
errors: row.errors ?? [],
|
|
677
|
+
report: row.report ?? [],
|
|
328
678
|
createdAt: new Date(row.created_at).toISOString(),
|
|
329
679
|
finishedAt: row.finished_at ? new Date(row.finished_at).toISOString() : null,
|
|
330
680
|
targets,
|
|
331
681
|
};
|
|
332
682
|
}
|
|
333
683
|
}
|
|
684
|
+
/** A row failure as the user reads it; database constraint errors get their Arabic text. */
|
|
685
|
+
function rowMessage(error) {
|
|
686
|
+
const code = error?.code;
|
|
687
|
+
if (code === '23505')
|
|
688
|
+
return translate('errors.E_DUPLICATE', 'ar');
|
|
689
|
+
if (code === '23503')
|
|
690
|
+
return translate('errors.E_RELATION', 'ar');
|
|
691
|
+
return error instanceof AdulaError || error instanceof Error
|
|
692
|
+
? error.message.slice(0, 300)
|
|
693
|
+
: 'خطأ غير معروف';
|
|
694
|
+
}
|