adula 2.0.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (214) hide show
  1. package/README.md +149 -18
  2. package/build/agent/AGENTS.template.md +4 -4
  3. package/build/agent/capabilities.md +22 -11
  4. package/build/agent/skills/adula-frontend-design/SKILL.md +1 -1
  5. package/build/agent/skills/idea-review/SKILL.md +2 -2
  6. package/build/commands/adula_outbox.d.ts +10 -0
  7. package/build/commands/adula_outbox.js +11 -0
  8. package/build/commands/adula_restore_reconcile.d.ts +10 -0
  9. package/build/commands/adula_restore_reconcile.js +34 -0
  10. package/build/commands/adula_runtime_health.d.ts +10 -0
  11. package/build/commands/adula_runtime_health.js +27 -0
  12. package/build/commands/adula_uploads_prune.d.ts +10 -0
  13. package/build/commands/adula_uploads_prune.js +16 -0
  14. package/build/commands/app.d.ts +10 -0
  15. package/build/commands/app.js +64 -0
  16. package/build/commands/backup_create.d.ts +11 -0
  17. package/build/commands/backup_create.js +99 -0
  18. package/build/commands/backup_restore_files.d.ts +11 -0
  19. package/build/commands/backup_restore_files.js +44 -0
  20. package/build/commands/backup_restore_test.d.ts +12 -0
  21. package/build/commands/backup_restore_test.js +185 -0
  22. package/build/commands/backup_services.d.ts +31 -0
  23. package/build/commands/backup_services.js +6 -0
  24. package/build/commands/backup_verify.d.ts +9 -0
  25. package/build/commands/backup_verify.js +59 -0
  26. package/build/commands/backup_verify_snapshot.d.ts +10 -0
  27. package/build/commands/backup_verify_snapshot.js +37 -0
  28. package/build/commands/client.d.ts +16 -0
  29. package/build/commands/client.js +71 -0
  30. package/build/commands/doctor.js +4 -1
  31. package/build/commands/install.d.ts +8 -0
  32. package/build/commands/install.js +63 -45
  33. package/build/commands/main.d.ts +3 -2
  34. package/build/commands/main.js +26 -0
  35. package/build/commands/module_add.js +4 -20
  36. package/build/commands/resource.d.ts +2 -0
  37. package/build/commands/resource.js +12 -3
  38. package/build/commands/resource_migration.d.ts +15 -0
  39. package/build/commands/resource_migration.js +72 -0
  40. package/build/commands/resource_snapshot.d.ts +2 -0
  41. package/build/commands/resource_snapshot.js +3 -0
  42. package/build/commands/sla_evaluate.d.ts +14 -0
  43. package/build/commands/sla_evaluate.js +16 -0
  44. package/build/commands/ui.js +5 -3
  45. package/build/commands/ui_css.d.ts +9 -0
  46. package/build/commands/ui_css.js +19 -0
  47. package/build/commands/upgrade.d.ts +1 -1
  48. package/build/commands/upgrade.js +1 -1
  49. package/build/configure.js +1 -1
  50. package/build/database/migrations/1770000000020_two_factor.d.ts +5 -0
  51. package/build/database/migrations/1770000000020_two_factor.js +10 -0
  52. package/build/database/migrations/1770000000041_import_modes.d.ts +5 -0
  53. package/build/database/migrations/1770000000041_import_modes.js +10 -0
  54. package/build/database/migrations/1770000000042_sync_jobs.d.ts +5 -0
  55. package/build/database/migrations/1770000000042_sync_jobs.js +10 -0
  56. package/build/database/migrations/1770000000043_notification_channels.d.ts +5 -0
  57. package/build/database/migrations/1770000000043_notification_channels.js +10 -0
  58. package/build/database/migrations/1770000000044_channel_confirmations.d.ts +5 -0
  59. package/build/database/migrations/1770000000044_channel_confirmations.js +10 -0
  60. package/build/database/migrations/1770000000045_import_leases.d.ts +5 -0
  61. package/build/database/migrations/1770000000045_import_leases.js +10 -0
  62. package/build/database/migrations/1770000000046_workflow_delegation_sla.d.ts +5 -0
  63. package/build/database/migrations/1770000000046_workflow_delegation_sla.js +10 -0
  64. package/build/database/migrations/1770000000049_print_templates.d.ts +5 -0
  65. package/build/database/migrations/1770000000049_print_templates.js +10 -0
  66. package/build/index.d.ts +56 -10
  67. package/build/index.js +34 -6
  68. package/build/providers/adula_provider.d.ts +2 -0
  69. package/build/providers/adula_provider.js +7 -0
  70. package/build/src/admin/contracts.js +1 -1
  71. package/build/src/admin/controller.d.ts +14 -1
  72. package/build/src/admin/controller.js +77 -13
  73. package/build/src/admin/errors.d.ts +32 -0
  74. package/build/src/admin/errors.js +63 -1
  75. package/build/src/admin/presentation.d.ts +21 -2
  76. package/build/src/admin/resource/editor.d.ts +59 -0
  77. package/build/src/admin/resource/editor.js +323 -0
  78. package/build/src/admin/resource/export.d.ts +37 -0
  79. package/build/src/admin/resource/export.js +181 -0
  80. package/build/src/admin/resource/lifecycle.d.ts +21 -0
  81. package/build/src/admin/resource/lifecycle.js +91 -0
  82. package/build/src/admin/resource/query.d.ts +100 -0
  83. package/build/src/admin/resource/query.js +508 -0
  84. package/build/src/admin/resource/read.d.ts +106 -0
  85. package/build/src/admin/resource/read.js +327 -0
  86. package/build/src/admin/resource/shared.d.ts +77 -0
  87. package/build/src/admin/resource/shared.js +222 -0
  88. package/build/src/admin/resource/trash.d.ts +10 -0
  89. package/build/src/admin/resource/trash.js +56 -0
  90. package/build/src/admin/resource/write.d.ts +57 -0
  91. package/build/src/admin/resource/write.js +574 -0
  92. package/build/src/admin/resource_service.d.ts +66 -193
  93. package/build/src/admin/resource_service.js +69 -1345
  94. package/build/src/attachments/attachment_service.d.ts +46 -2
  95. package/build/src/attachments/attachment_service.js +96 -4
  96. package/build/src/auth/conditions.d.ts +5 -0
  97. package/build/src/auth/conditions.js +26 -3
  98. package/build/src/auth/sql.js +17 -5
  99. package/build/src/auth/two_factor.d.ts +96 -0
  100. package/build/src/auth/two_factor.js +259 -0
  101. package/build/src/collaboration/assignments.d.ts +36 -1
  102. package/build/src/collaboration/assignments.js +132 -66
  103. package/build/src/collaboration/record_collaboration.d.ts +2 -3
  104. package/build/src/collaboration/record_collaboration.js +19 -16
  105. package/build/src/collaboration/shared.d.ts +16 -0
  106. package/build/src/collaboration/shared.js +21 -0
  107. package/build/src/commands/app_scaffold.d.ts +72 -0
  108. package/build/src/commands/app_scaffold.js +285 -0
  109. package/build/src/commands/capabilities.js +34 -8
  110. package/build/src/commands/doctor.d.ts +14 -2
  111. package/build/src/commands/doctor.js +50 -5
  112. package/build/src/commands/gap_report.d.ts +1 -1
  113. package/build/src/commands/gap_report.js +2 -2
  114. package/build/src/commands/generator.d.ts +37 -1
  115. package/build/src/commands/generator.js +61 -18
  116. package/build/src/commands/snapshot.d.ts +10 -13
  117. package/build/src/commands/snapshot.js +31 -14
  118. package/build/src/commands/source_markers.d.ts +10 -0
  119. package/build/src/commands/source_markers.js +29 -0
  120. package/build/src/core/activity.d.ts +2 -0
  121. package/build/src/core/activity.js +7 -1
  122. package/build/src/core/administrators.d.ts +38 -0
  123. package/build/src/core/administrators.js +129 -0
  124. package/build/src/core/html_sanitizer.d.ts +20 -0
  125. package/build/src/core/html_sanitizer.js +261 -0
  126. package/build/src/core/like.d.ts +5 -0
  127. package/build/src/core/like.js +5 -0
  128. package/build/src/core/lookups.d.ts +80 -0
  129. package/build/src/core/lookups.js +319 -0
  130. package/build/src/core/message_templates.d.ts +7 -2
  131. package/build/src/core/message_templates.js +72 -11
  132. package/build/src/core/module_seed.js +2 -1
  133. package/build/src/core/notification_channels.d.ts +235 -0
  134. package/build/src/core/notification_channels.js +660 -0
  135. package/build/src/core/roles.js +2 -2
  136. package/build/src/core/saved_views.d.ts +8 -1
  137. package/build/src/core/saved_views.js +36 -2
  138. package/build/src/core/sequences.d.ts +32 -0
  139. package/build/src/core/sequences.js +97 -0
  140. package/build/src/core/settings.d.ts +5 -0
  141. package/build/src/core/settings.js +19 -2
  142. package/build/src/core/template_language.d.ts +61 -0
  143. package/build/src/core/template_language.js +141 -0
  144. package/build/src/core/ui_preferences.d.ts +4 -0
  145. package/build/src/core/ui_preferences.js +4 -1
  146. package/build/src/core/user_invitations.d.ts +5 -1
  147. package/build/src/core/user_invitations.js +7 -1
  148. package/build/src/core/users.js +1 -1
  149. package/build/src/database/calendar_days.d.ts +19 -0
  150. package/build/src/database/calendar_days.js +36 -0
  151. package/build/src/database/schema.d.ts +155 -2
  152. package/build/src/database/schema.js +531 -59
  153. package/build/src/database/schema_diff.d.ts +56 -0
  154. package/build/src/database/schema_diff.js +157 -0
  155. package/build/src/events/record_mutation.d.ts +2 -1
  156. package/build/src/events/record_mutation.js +1 -0
  157. package/build/src/integrations/api_client.d.ts +37 -0
  158. package/build/src/integrations/api_client.js +552 -0
  159. package/build/src/integrations/import_adapters.d.ts +37 -0
  160. package/build/src/integrations/import_adapters.js +35 -0
  161. package/build/src/integrations/imports.d.ts +118 -16
  162. package/build/src/integrations/imports.js +406 -73
  163. package/build/src/integrations/openapi.d.ts +3 -1
  164. package/build/src/integrations/openapi.js +454 -31
  165. package/build/src/integrations/outbound.d.ts +51 -0
  166. package/build/src/integrations/outbound.js +195 -0
  167. package/build/src/integrations/print.d.ts +21 -1
  168. package/build/src/integrations/print.js +63 -21
  169. package/build/src/integrations/print_templates.d.ts +216 -0
  170. package/build/src/integrations/print_templates.js +1015 -0
  171. package/build/src/integrations/sync_jobs.d.ts +139 -0
  172. package/build/src/integrations/sync_jobs.js +416 -0
  173. package/build/src/integrations/webhooks.d.ts +12 -12
  174. package/build/src/integrations/webhooks.js +34 -43
  175. package/build/src/languages/ar.d.ts +11 -0
  176. package/build/src/languages/ar.js +336 -0
  177. package/build/src/languages/ar_ui.d.ts +332 -0
  178. package/build/src/languages/ar_ui.js +333 -0
  179. package/build/src/languages/catalog.d.ts +66 -0
  180. package/build/src/languages/catalog.js +152 -0
  181. package/build/src/languages/en.d.ts +7 -0
  182. package/build/src/languages/en.js +330 -0
  183. package/build/src/languages/en_ui.d.ts +3 -0
  184. package/build/src/languages/en_ui.js +328 -0
  185. package/build/src/languages/validation.d.ts +26 -0
  186. package/build/src/languages/validation.js +60 -0
  187. package/build/src/lists/bulk_actions.d.ts +89 -0
  188. package/build/src/lists/bulk_actions.js +268 -0
  189. package/build/src/lists/dashboards.d.ts +96 -0
  190. package/build/src/lists/dashboards.js +241 -0
  191. package/build/src/mcp/tools.js +1 -1
  192. package/build/src/resource/define_resource.d.ts +18 -1
  193. package/build/src/resource/define_resource.js +205 -18
  194. package/build/src/resource/definition.d.ts +44 -0
  195. package/build/src/resource/definition.js +171 -0
  196. package/build/src/resource/registry.js +15 -4
  197. package/build/src/resource/types.d.ts +304 -16
  198. package/build/src/resource/values.d.ts +56 -1
  199. package/build/src/resource/values.js +237 -5
  200. package/build/src/services/settings.d.ts +2 -0
  201. package/build/src/services/settings.js +5 -1
  202. package/build/src/upgrade/legacy.js +2 -2
  203. package/build/src/workflows/calendar.d.ts +9 -0
  204. package/build/src/workflows/calendar.js +44 -0
  205. package/build/src/workflows/define_workflow.d.ts +48 -3
  206. package/build/src/workflows/define_workflow.js +57 -13
  207. package/build/src/workflows/delegations.d.ts +93 -0
  208. package/build/src/workflows/delegations.js +287 -0
  209. package/build/src/workflows/engine.d.ts +69 -4
  210. package/build/src/workflows/engine.js +441 -83
  211. package/build/src/workflows/sla.d.ts +15 -0
  212. package/build/src/workflows/sla.js +8 -0
  213. package/build/stubs/resource_contract.txt +12 -1
  214. package/package.json +5 -2
@@ -1,17 +1,25 @@
1
+ import { randomUUID } from 'node:crypto';
1
2
  import Papa from 'papaparse';
2
3
  import { subject } from '@casl/ability';
3
4
  import { buildAbility } from '../auth/ability.js';
5
+ import { columnName } from '../resource/define_resource.js';
4
6
  import { AdulaError } from '../admin/errors.js';
5
7
  import { notifyWithTemplate } from '../core/message_templates.js';
8
+ import { translate } from '../languages/catalog.js';
9
+ import { csvImportAdapter, extensionOf, importFile, } from './import_adapters.js';
6
10
  export const IMPORT_ROW_LIMIT = 5000;
7
11
  const COLUMN_LIMIT = 100;
8
12
  const ERROR_LIMIT = 200;
9
13
  const FILE_LIMIT = 5 * 1024 * 1024;
10
- const UNSUPPORTED = new Set(['hasMany', 'attachment', 'json']);
14
+ // Lists (multiSelect, manyToMany) have no single-cell CSV form yet.
15
+ const UNSUPPORTED = new Set(['hasMany', 'attachment', 'json', 'multiSelect', 'manyToMany']);
16
+ const MODES = new Set(['create', 'upsert']);
11
17
  const normalize = (value) => value
12
18
  .trim()
13
19
  .toLowerCase()
14
20
  .replace(/[\s_-]+/g, '');
21
+ /** Template headers mark required columns with `*`; the mark never blocks matching. */
22
+ const headerKey = (value) => normalize(value.replace(/\s*\*+\s*$/, ''));
15
23
  /** Converts one CSV cell to the field's stored representation, or throws a readable error. */
16
24
  export function importCell(field, raw, lookups) {
17
25
  const value = raw.trim();
@@ -62,6 +70,14 @@ export function importCell(field, raw, lookups) {
62
70
  throw new Error('تاريخ ووقت غير صالحين');
63
71
  return date.toISOString();
64
72
  }
73
+ case 'decimal':
74
+ case 'percent': {
75
+ // "," and "٬" only as thousands grouping (1,234.5); "٫" is the Arabic decimal point.
76
+ const plain = latin.replace(/[\s%]/g, '').replace('٫', '.').replace(/٬/g, ',');
77
+ if (!/^-?(\d{1,3}(,\d{3})+|\d+)(\.\d+)?$/.test(plain))
78
+ throw new Error('رقم عشري غير صالح');
79
+ return plain.replace(/,/g, '');
80
+ }
65
81
  case 'lookup': {
66
82
  const key = lookups.get(normalize(value));
67
83
  if (!key)
@@ -72,28 +88,62 @@ export function importCell(field, raw, lookups) {
72
88
  return value;
73
89
  }
74
90
  }
91
+ /** How long a claimed batch stays reserved for one worker; renewed after every row. */
92
+ const LEASE_SECONDS = 300;
93
+ const leaseExpiry = (db) => db.raw(`now() + interval '${LEASE_SECONDS} seconds'`);
94
+ /** Raised inside a dry-run transaction so it always rolls back. */
95
+ class DryRunRollback extends Error {
96
+ }
75
97
  /**
76
- * CSV imports in batches: the upload is parsed and stored, the user maps columns
77
- * to writable fields, and a worker saves each row through ResourceService with
78
- * the importing user's current permissions. Row failures never stop the batch.
98
+ * File imports in batches: the upload is parsed by a format adapter (CSV built in) and
99
+ * stored, the user maps columns to writable fields, and a worker saves each row through
100
+ * ResourceService with the importing user's current permissions. Row failures never stop
101
+ * the batch. A dry run validates every row exactly like the real import inside a
102
+ * transaction that is rolled back, so nothing (records, activity, outbox events, sequence
103
+ * numbers, notifications) is kept; the batch then waits as 'validated' for the real run.
104
+ * Upsert mode updates the live record matched by a unique field and creates the rest.
79
105
  */
80
106
  export class ImportBatches {
81
107
  db;
82
108
  registry;
83
109
  resources;
84
110
  actors;
85
- constructor(db, registry, resources, actors) {
111
+ formats = [csvImportAdapter];
112
+ constructor(db, registry, resources, actors, options = {}) {
86
113
  this.db = db;
87
114
  this.registry = registry;
88
115
  this.resources = resources;
89
116
  this.actors = actors;
117
+ for (const adapter of options.adapters ?? [])
118
+ this.registerAdapter(adapter);
119
+ }
120
+ /**
121
+ * Adds a file format, for example an XLSX adapter. A later adapter with the same
122
+ * name replaces the earlier one; CSV stays the fallback for text without an extension.
123
+ */
124
+ registerAdapter(adapter) {
125
+ if (!/^[a-z0-9][a-z0-9_-]*$/.test(adapter.name) || !adapter.extensions.length)
126
+ throw new Error('An import adapter needs a lower-case name and at least one extension');
127
+ this.formats = [...this.formats.filter((item) => item.name !== adapter.name), adapter];
128
+ return this;
129
+ }
130
+ /** Registered formats, CSV first. */
131
+ adapters() {
132
+ return this.formats;
133
+ }
134
+ /** Every accepted upload extension, for the upload validator and the file picker. */
135
+ extensions() {
136
+ return [...new Set(this.formats.flatMap((adapter) => adapter.extensions))];
90
137
  }
91
- /** Writable, importable fields for this actor. */
92
- targets(name, actor) {
138
+ /** Writable, importable fields for this actor; upsert also needs update permission. */
139
+ targets(name, actor, mode = 'create') {
93
140
  const resource = this.registry.get(name);
94
141
  const ability = buildAbility(actor.rules, this.registry.all());
95
142
  if (!resource.actions.includes('create') || !ability.can('create', name))
96
143
  throw new AdulaError(403, 'E_FORBIDDEN', 'ليس لديك صلاحية الإضافة إلى هذا الكيان');
144
+ const upsert = mode === 'upsert';
145
+ if (upsert && (!resource.actions.includes('update') || !ability.can('update', name)))
146
+ throw new AdulaError(403, 'E_FORBIDDEN', 'الإضافة أو التحديث يتطلب صلاحية التعديل أيضاً');
97
147
  const fields = resource.form
98
148
  .filter((key) => {
99
149
  const field = resource.fields[key];
@@ -101,13 +151,15 @@ export class ImportBatches {
101
151
  !field.sequence &&
102
152
  actor.permissionLevel >=
103
153
  Math.max(field.permissionLevel ?? 0, resource.hidden?.includes(key) ? 1 : 0) &&
104
- ability.can('create', subject(name, {}), key));
154
+ ability.can('create', subject(name, {}), key) &&
155
+ (!upsert || ability.can('update', subject(name, {}), key)));
105
156
  })
106
157
  .map((key) => ({
107
158
  key,
108
159
  label: resource.fields[key].label.ar,
109
160
  type: resource.fields[key].type,
110
161
  required: Boolean(resource.fields[key].required),
162
+ unique: Boolean(resource.fields[key].unique),
111
163
  }));
112
164
  // An inherited scope takes the unit from the parent row instead (#45).
113
165
  if (resource.scoped && !resource.scope)
@@ -116,29 +168,52 @@ export class ImportBatches {
116
168
  label: 'الوحدة التنظيمية (رقم)',
117
169
  type: 'integer',
118
170
  required: true,
171
+ unique: false,
119
172
  });
120
173
  return fields;
121
174
  }
175
+ /**
176
+ * A CSV header row of the actor's importable fields in Arabic, required ones marked
177
+ * with `*`, with a UTF-8 byte-order mark so spreadsheet programs show Arabic. The
178
+ * headers map automatically when the filled template is uploaded.
179
+ */
180
+ template(name, actor, mode = 'create') {
181
+ const targets = this.targets(name, actor, mode);
182
+ const headers = targets.map((target) => `${target.label}${target.required ? ' *' : ''}`);
183
+ return {
184
+ fileName: `${name}-template.csv`,
185
+ content: `\uFEFF${Papa.unparse([headers])}\n`,
186
+ };
187
+ }
122
188
  async create(name, actor, input) {
123
- const targets = this.targets(name, actor);
189
+ const mode = this.mode(input.mode ?? 'create');
190
+ const targets = this.targets(name, actor, mode);
191
+ const matchField = input.matchField ? this.matchField(targets, input.matchField) : null;
124
192
  const fileName = typeof input.fileName === 'string' ? input.fileName.slice(0, 200) : 'import.csv';
125
- if (typeof input.content !== 'string' || !input.content.trim())
193
+ let bytes;
194
+ if (typeof input.content === 'string') {
195
+ if (!input.content.trim())
196
+ throw new AdulaError(422, 'E_IMPORT_FILE', 'الملف فارغ');
197
+ bytes = Buffer.from(input.content, input.encoding === 'base64' ? 'base64' : 'utf8');
198
+ }
199
+ else if (input.content instanceof Uint8Array)
200
+ bytes = input.content;
201
+ else
202
+ throw new AdulaError(422, 'E_IMPORT_FILE', 'الملف فارغ');
203
+ if (!bytes.byteLength)
126
204
  throw new AdulaError(422, 'E_IMPORT_FILE', 'الملف فارغ');
127
- if (Buffer.byteLength(input.content) > FILE_LIMIT)
205
+ if (bytes.byteLength > FILE_LIMIT)
128
206
  throw new AdulaError(422, 'E_IMPORT_FILE', 'حجم الملف يتجاوز 5 ميغابايت');
129
- const parsed = Papa.parse(input.content.replace(/^\uFEFF/, ''), {
130
- skipEmptyLines: 'greedy',
131
- });
132
- if (parsed.errors.some((error) => error.type !== 'Delimiter'))
133
- throw new AdulaError(422, 'E_IMPORT_FILE', `تعذر قراءة CSV: ${parsed.errors[0].message}`);
134
- const [headers = [], ...rows] = parsed.data;
207
+ const file = importFile(fileName, bytes, typeof input.mimeType === 'string' ? input.mimeType : undefined);
208
+ const adapter = this.adapterFor(file, input.format, typeof input.content === 'string');
209
+ const { headers, rows } = await adapter.parse(file);
135
210
  if (!headers.length || headers.length > COLUMN_LIMIT)
136
211
  throw new AdulaError(422, 'E_IMPORT_FILE', 'يجب أن يحتوي الملف صف عناوين (100 عمود كحد أقصى)');
137
212
  if (!rows.length || rows.length > IMPORT_ROW_LIMIT)
138
213
  throw new AdulaError(422, 'E_IMPORT_ROWS', `عدد الصفوف بين 1 و${IMPORT_ROW_LIMIT}`);
139
214
  const mapping = {};
140
215
  headers.forEach((header, index) => {
141
- const target = targets.find((field) => [field.key, field.label].some((alias) => normalize(alias) === normalize(header)));
216
+ const target = targets.find((field) => [field.key, field.label].some((alias) => normalize(alias) === headerKey(String(header))));
142
217
  if (target && !Object.values(mapping).includes(target.key))
143
218
  mapping[String(index)] = target.key;
144
219
  });
@@ -148,9 +223,12 @@ export class ImportBatches {
148
223
  user_id: actor.id,
149
224
  file_name: fileName,
150
225
  status: 'mapping',
151
- headers: JSON.stringify(headers),
226
+ mode,
227
+ match_field: matchField,
228
+ dry_run: input.dryRun === true,
229
+ headers: JSON.stringify(headers.map(String)),
152
230
  mapping: JSON.stringify(mapping),
153
- rows: JSON.stringify(rows.map((cells) => cells.slice(0, headers.length))),
231
+ rows: JSON.stringify(rows.map((cells) => cells.slice(0, headers.length).map((cell) => String(cell ?? '')))),
154
232
  total: rows.length,
155
233
  })
156
234
  .returning('*');
@@ -158,22 +236,27 @@ export class ImportBatches {
158
236
  }
159
237
  async show(id, actor) {
160
238
  const row = await this.owned(id, actor);
161
- return this.present(row, this.safeTargets(row.resource, actor));
239
+ return this.present(row, this.safeTargets(row.resource, actor, row.mode));
162
240
  }
163
241
  async list(actor) {
164
242
  const rows = await this.db('import_batches')
165
243
  .where('user_id', actor.id)
166
244
  .orderBy('id', 'desc')
167
245
  .limit(50)
168
- .select('id', 'resource', 'file_name', 'status', 'headers', 'mapping', 'total', 'processed', 'created', 'failed', 'errors', 'created_at', 'finished_at');
169
- return rows.map((row) => this.present({ ...row, rows: [] }, []));
246
+ .select('id', 'resource', 'file_name', 'status', 'mode', 'match_field', 'dry_run', 'headers', 'mapping', 'total', 'processed', 'created', 'updated', 'failed', 'errors', 'created_at', 'finished_at');
247
+ return rows.map((row) => this.present({ ...row, rows: [], report: [] }, []));
170
248
  }
171
- /** Stores the column mapping and queues the batch for the worker. */
172
- async start(id, actor, mapping) {
249
+ /**
250
+ * Stores the column mapping and queues the batch for the worker. A batch in 'mapping'
251
+ * runs as created (dry run or not) unless `dryRun` says otherwise; a 'validated' batch
252
+ * runs for real by default, so the same batch is imported after its dry run.
253
+ */
254
+ async start(id, actor, mapping, options = {}) {
173
255
  const row = await this.owned(id, actor);
174
- if (row.status !== 'mapping')
256
+ if (!['mapping', 'validated'].includes(row.status))
175
257
  throw new AdulaError(409, 'E_IMPORT_STATE', 'بدأت معالجة هذه الدفعة بالفعل');
176
- const targets = this.targets(row.resource, actor);
258
+ const mode = this.mode(options.mode ?? row.mode ?? 'create');
259
+ const targets = this.targets(row.resource, actor, mode);
177
260
  if (!mapping || typeof mapping !== 'object' || Array.isArray(mapping))
178
261
  throw new AdulaError(422, 'E_IMPORT_MAPPING', 'مطابقة الأعمدة غير صالحة');
179
262
  const clean = {};
@@ -184,6 +267,10 @@ export class ImportBatches {
184
267
  const index = Number(column);
185
268
  if (!Number.isInteger(index) || index < 0 || index >= row.headers.length)
186
269
  throw new AdulaError(422, 'E_IMPORT_MAPPING', 'عمود غير موجود');
270
+ // Files cannot come from a spreadsheet cell: they are uploaded on the record page.
271
+ const named = this.registry.get(row.resource).fields;
272
+ if (typeof key === 'string' && Object.hasOwn(named, key) && named[key].type === 'attachment')
273
+ throw new AdulaError(422, 'E_IMPORT_MAPPING', `لا يمكن استيراد المرفقات من ملف؛ ارفعها من صفحة السجل: ${named[key].label.ar}`);
187
274
  if (typeof key !== 'string' || !targets.some((target) => target.key === key) || used.has(key))
188
275
  throw new AdulaError(422, 'E_IMPORT_MAPPING', `حقل غير قابل للاستيراد أو مكرر: ${String(key)}`);
189
276
  used.add(key);
@@ -192,19 +279,60 @@ export class ImportBatches {
192
279
  const missing = targets.filter((target) => target.required && !used.has(target.key));
193
280
  if (missing.length)
194
281
  throw new AdulaError(422, 'E_IMPORT_MAPPING', `حقول مطلوبة بلا عمود: ${missing.map((target) => target.label).join('، ')}`);
195
- await this.db('import_batches')
196
- .where('id', row.id)
197
- .update({ mapping: JSON.stringify(clean), status: 'queued' });
282
+ let matchField = null;
283
+ if (mode === 'upsert') {
284
+ matchField = this.matchField(targets, options.matchField ?? row.match_field);
285
+ if (!used.has(matchField))
286
+ throw new AdulaError(422, 'E_IMPORT_MATCH', 'اربط حقل المطابقة بعمود من الملف');
287
+ }
288
+ const dryRun = typeof options.dryRun === 'boolean'
289
+ ? options.dryRun
290
+ : row.status === 'validated'
291
+ ? false
292
+ : Boolean(row.dry_run);
293
+ // Conditional on the state read above: of two concurrent starts only one queues the
294
+ // batch, and a batch a worker already claimed is never reset to row 0.
295
+ const queued = await this.db('import_batches')
296
+ .where({ id: row.id, status: row.status })
297
+ .whereIn('status', ['mapping', 'validated'])
298
+ .update({
299
+ mapping: JSON.stringify(clean),
300
+ mode,
301
+ match_field: matchField,
302
+ dry_run: dryRun,
303
+ status: 'queued',
304
+ processed: 0,
305
+ created: 0,
306
+ updated: 0,
307
+ failed: 0,
308
+ errors: JSON.stringify([]),
309
+ report: JSON.stringify([]),
310
+ started_at: null,
311
+ finished_at: null,
312
+ lease_id: null,
313
+ lease_expires_at: null,
314
+ });
315
+ if (!queued)
316
+ throw new AdulaError(409, 'E_IMPORT_STATE', 'بدأت معالجة هذه الدفعة بالفعل');
198
317
  return this.show(row.id, actor);
199
318
  }
200
319
  /**
201
- * Worker step: claims one queued or running batch and imports up to `chunk`
202
- * rows. Progress is committed per row, so a crash resumes after the last one.
320
+ * Worker step: claims one queued or running batch and imports (or, for a dry run,
321
+ * validates) up to `chunk` rows. The claim is a lease (lease_id, lease_expires_at, renewed
322
+ * after every row): concurrent workers never share a batch, and an expired lease is taken
323
+ * over, resuming after the last committed row. Import progress is committed per row in
324
+ * the row's own transaction; a dry run records its progress and report once per chunk.
325
+ * Returns null when no batch is free, or when this worker's lease was taken over (it then
326
+ * writes nothing further).
203
327
  */
204
328
  async process(chunk = 200) {
329
+ const lease = randomUUID();
205
330
  const batch = await this.db.transaction(async (trx) => {
331
+ // A batch is free when no worker holds an unexpired lease on it; an expired lease
332
+ // (a crashed or stalled worker) is taken over and processing resumes after its last row.
206
333
  const row = await trx('import_batches')
207
334
  .whereIn('status', ['queued', 'running'])
335
+ .where((free) => free.whereNull('lease_id').orWhere('lease_expires_at', '<', trx.fn.now()))
208
336
  .orderBy('id')
209
337
  .forUpdate()
210
338
  .skipLocked()
@@ -213,11 +341,19 @@ export class ImportBatches {
213
341
  return null;
214
342
  await trx('import_batches')
215
343
  .where('id', row.id)
216
- .update({ status: 'running', started_at: row.started_at ?? trx.fn.now() });
344
+ .update({
345
+ status: 'running',
346
+ started_at: row.started_at ?? trx.fn.now(),
347
+ lease_id: lease,
348
+ lease_expires_at: leaseExpiry(trx),
349
+ });
217
350
  return row;
218
351
  });
219
352
  if (!batch)
220
353
  return null;
354
+ // Every later write names this lease; one that affects no row means another worker
355
+ // took the batch over, and this worker stops without writing anything else.
356
+ const owned = (db) => db('import_batches').where({ id: batch.id, lease_id: lease });
221
357
  const actor = await this.actors.load(Number(batch.user_id));
222
358
  const resource = this.registry.get(batch.resource);
223
359
  const lookupGroups = new Map();
@@ -235,62 +371,243 @@ export class ImportBatches {
235
371
  }
236
372
  const mapping = batch.mapping;
237
373
  const rows = batch.rows;
238
- let { processed, created, failed } = batch;
239
- const errors = [...batch.errors];
374
+ const dryRun = Boolean(batch.dry_run);
375
+ const mode = batch.mode === 'upsert' ? 'upsert' : 'create';
376
+ const matchField = mode === 'upsert' ? batch.match_field : null;
377
+ // A dry run rolls each row back, so it finds duplicates within the file itself.
378
+ const earlier = dryRun ? this.earlierDuplicates(resource, mapping, rows, matchField) : null;
379
+ let { processed, created, updated, failed } = batch;
380
+ let errors = [...batch.errors];
381
+ const report = [...(batch.report ?? [])];
240
382
  const end = Math.min(rows.length, processed + chunk);
241
383
  for (let index = processed; index < end; index++) {
242
384
  const cells = rows[index];
243
- const input = {};
244
- try {
245
- for (const [column, key] of Object.entries(mapping)) {
246
- const raw = cells[Number(column)] ?? '';
247
- if (key === 'orgUnitId') {
248
- input.orgUnitId = raw.trim() === '' ? null : Number(raw.trim());
249
- continue;
250
- }
251
- try {
252
- input[key] = importCell(resource.fields[key], raw, lookupGroups.get(key) ?? new Map());
253
- }
254
- catch (error) {
255
- throw new Error(`${resource.fields[key].label.ar}: ${error.message}`);
385
+ // One row: convert, then save (in a savepoint of `trx` for a real import).
386
+ const importRow = async (trx) => {
387
+ const input = {};
388
+ try {
389
+ for (const [column, key] of Object.entries(mapping)) {
390
+ const raw = cells[Number(column)] ?? '';
391
+ if (key === 'orgUnitId') {
392
+ input.orgUnitId = raw.trim() === '' ? null : Number(raw.trim());
393
+ continue;
394
+ }
395
+ try {
396
+ input[key] = importCell(resource.fields[key], raw, lookupGroups.get(key) ?? new Map());
397
+ }
398
+ catch (error) {
399
+ throw new Error(`${resource.fields[key].label.ar}: ${error.message}`);
400
+ }
256
401
  }
402
+ const duplicate = earlier?.duplicates.get(index);
403
+ if (duplicate)
404
+ throw new Error(duplicate);
405
+ let action = await this.saveRow(resource, actor, input, matchField, dryRun, trx);
406
+ // A repeated match value updates the record the earlier row of the file created.
407
+ if (earlier?.repeats.has(index))
408
+ action = 'update';
409
+ return { row: index + 2, ok: true, action };
410
+ }
411
+ catch (error) {
412
+ return { row: index + 2, ok: false, message: rowMessage(error) };
257
413
  }
258
- await this.resources.save(batch.resource, actor, input);
259
- created++;
414
+ };
415
+ const count = (outcome) => {
416
+ const next = { created, updated, failed, errors };
417
+ if (!outcome.ok) {
418
+ next.failed++;
419
+ if (errors.length < ERROR_LIMIT)
420
+ next.errors = [...errors, { row: outcome.row, message: outcome.message ?? '' }];
421
+ }
422
+ else if (outcome.action === 'create')
423
+ next.created++;
424
+ else
425
+ next.updated++;
426
+ return next;
427
+ };
428
+ let next;
429
+ if (dryRun) {
430
+ const outcome = await importRow();
431
+ next = count(outcome);
432
+ report.push(outcome);
433
+ // A dry run saves nothing: only the lease is renewed per row; progress and the
434
+ // report are written once per chunk, and a crash only validates that chunk again.
435
+ if (!(await owned(this.db).update({ lease_expires_at: leaseExpiry(this.db) })))
436
+ return null;
260
437
  }
261
- catch (error) {
262
- failed++;
263
- if (errors.length < ERROR_LIMIT)
264
- errors.push({
265
- row: index + 2,
266
- message: error instanceof AdulaError || error instanceof Error
267
- ? error.message.slice(0, 300)
268
- : 'خطأ غير معروف',
438
+ else {
439
+ // A real import saves the row and its progress in one transaction that first locks
440
+ // the batch under this lease at this row, so every row is imported exactly once
441
+ // even when an expired lease was taken over by another worker meanwhile.
442
+ next = await this.db.transaction(async (trx) => {
443
+ if (!(await owned(trx).where('processed', index).forUpdate().first('id')))
444
+ return null;
445
+ const counts = count(await importRow(trx));
446
+ await owned(trx).update({
447
+ processed: index + 1,
448
+ created: counts.created,
449
+ updated: counts.updated,
450
+ failed: counts.failed,
451
+ errors: JSON.stringify(counts.errors),
452
+ lease_expires_at: leaseExpiry(trx),
269
453
  });
454
+ return counts;
455
+ });
456
+ if (!next)
457
+ return null;
270
458
  }
459
+ ;
460
+ ({ created, updated, failed, errors } = next);
271
461
  processed = index + 1;
272
- await this.db('import_batches')
273
- .where('id', batch.id)
274
- .update({ processed, created, failed, errors: JSON.stringify(errors) });
275
462
  }
276
- if (processed >= rows.length) {
277
- await this.db.transaction(async (trx) => {
278
- await trx('import_batches')
279
- .where('id', batch.id)
463
+ const finished = processed >= rows.length;
464
+ if (dryRun) {
465
+ const written = await owned(this.db).update({
466
+ processed,
467
+ created,
468
+ updated,
469
+ failed,
470
+ errors: JSON.stringify(errors),
471
+ report: JSON.stringify(report),
472
+ // The rows stay for the real import; a dry run sends no notification.
473
+ ...(finished ? { status: 'validated', finished_at: this.db.fn.now() } : {}),
474
+ lease_id: null,
475
+ lease_expires_at: null,
476
+ });
477
+ if (!written)
478
+ return null;
479
+ }
480
+ else if (finished) {
481
+ const written = await this.db.transaction(async (trx) => {
482
+ const count = await owned(trx)
280
483
  // Row data is only needed while importing; the log keeps counts and errors.
281
- .update({ status: 'done', finished_at: trx.fn.now(), rows: JSON.stringify([]) });
282
- await notifyWithTemplate(trx, Number(batch.user_id), 'import.finished', {
283
- resource: resource.label.ar,
284
- created,
285
- failed,
484
+ .update({
485
+ status: 'done',
486
+ finished_at: trx.fn.now(),
487
+ rows: JSON.stringify([]),
488
+ lease_id: null,
489
+ lease_expires_at: null,
286
490
  });
491
+ if (count)
492
+ await notifyWithTemplate(trx, Number(batch.user_id), 'import.finished', {
493
+ resource: resource.label.ar,
494
+ created: created + updated,
495
+ failed,
496
+ });
497
+ return count;
287
498
  });
499
+ if (!written)
500
+ return null;
288
501
  }
289
- return { id: Number(batch.id), processed, created, failed };
502
+ else if (!(await owned(this.db).update({ lease_id: null, lease_expires_at: null })))
503
+ // The chunk is done; release the batch so the next tick (any worker) continues it.
504
+ return null;
505
+ return { id: Number(batch.id), processed, created, updated, failed };
290
506
  }
291
- safeTargets(name, actor) {
507
+ /**
508
+ * Saves one row through ResourceService. Upsert first looks for the live record with
509
+ * the same match value; ResourceService then authorizes the update against the actor's
510
+ * scope and rules and checks the record's current `version`. A real import runs in a
511
+ * savepoint of `trx` (the caller's progress transaction); a dry run does the same inside
512
+ * a transaction that is always rolled back.
513
+ */
514
+ async saveRow(resource, actor, input, matchField, dryRun, trx) {
515
+ const work = async (scope) => {
516
+ let id;
517
+ if (matchField) {
518
+ const field = resource.fields[matchField];
519
+ const value = input[matchField];
520
+ if (value === null || value === undefined)
521
+ throw new Error(`${field.label.ar}: قيمة المطابقة مطلوبة`);
522
+ const found = await scope(resource.name)
523
+ .whereNull('deleted_at')
524
+ .where(field.column ?? columnName(matchField), value)
525
+ .first(['id', ...(resource.version ? ['version'] : [])]);
526
+ if (found) {
527
+ id = Number(found.id);
528
+ if (resource.version)
529
+ input.version = Number(found.version);
530
+ // An empty cell keeps the stored value of a matched record instead of clearing it.
531
+ for (const key of Object.keys(input))
532
+ if (input[key] === null)
533
+ delete input[key];
534
+ }
535
+ }
536
+ await this.resources.save(resource.name, actor, input, id, scope);
537
+ return id === undefined ? 'create' : 'update';
538
+ };
539
+ if (!dryRun)
540
+ return (trx ?? this.db).transaction(work);
541
+ let action = 'create';
292
542
  try {
293
- return this.targets(name, actor);
543
+ await this.db.transaction(async (scope) => {
544
+ action = await work(scope);
545
+ throw new DryRunRollback();
546
+ });
547
+ }
548
+ catch (error) {
549
+ if (!(error instanceof DryRunRollback))
550
+ throw error;
551
+ }
552
+ return action;
553
+ }
554
+ /**
555
+ * Rows whose unique value repeats an earlier row of the same file. In upsert mode a
556
+ * repeated match value is not an error: that row would update the earlier row's record.
557
+ */
558
+ earlierDuplicates(resource, mapping, rows, matchField) {
559
+ const duplicates = new Map();
560
+ const repeats = new Set();
561
+ for (const [column, key] of Object.entries(mapping)) {
562
+ if (!resource.fields[key]?.unique)
563
+ continue;
564
+ const seen = new Map();
565
+ rows.forEach((cells, index) => {
566
+ const value = (cells[Number(column)] ?? '').trim();
567
+ if (!value)
568
+ return;
569
+ const first = seen.get(value);
570
+ if (first === undefined)
571
+ seen.set(value, index);
572
+ else if (key === matchField)
573
+ repeats.add(index);
574
+ else if (!duplicates.has(index))
575
+ duplicates.set(index, `${resource.fields[key].label.ar}: القيمة مكررة في الملف (الصف ${first + 2})`);
576
+ });
577
+ }
578
+ return { duplicates, repeats };
579
+ }
580
+ adapterFor(file, format, text) {
581
+ if (format !== undefined && format !== null && format !== '') {
582
+ const named = this.formats.find((adapter) => adapter.name === format);
583
+ if (!named)
584
+ throw new AdulaError(422, 'E_IMPORT_FORMAT', 'صيغة الملف غير مدعومة');
585
+ return named;
586
+ }
587
+ const byExtension = this.formats.filter((adapter) => adapter.extensions.includes(extensionOf(file.fileName)));
588
+ const detected = [...byExtension].reverse().find((adapter) => adapter.detect(file)) ??
589
+ [...this.formats].reverse().find((adapter) => adapter.detect(file));
590
+ if (detected)
591
+ return detected;
592
+ // Text without a recognizable name keeps the historical CSV behaviour.
593
+ if (text && !extensionOf(file.fileName))
594
+ return csvImportAdapter;
595
+ throw new AdulaError(422, 'E_IMPORT_FORMAT', 'صيغة الملف غير مدعومة');
596
+ }
597
+ mode(value) {
598
+ if (typeof value !== 'string' || !MODES.has(value))
599
+ throw new AdulaError(422, 'E_IMPORT_MODE', 'نمط الاستيراد غير صالح');
600
+ return value;
601
+ }
602
+ matchField(targets, value) {
603
+ const target = targets.find((item) => item.key === value);
604
+ if (!target || !target.unique)
605
+ throw new AdulaError(422, 'E_IMPORT_MATCH', 'حقل المطابقة يجب أن يكون حقلاً فريداً قابلاً للاستيراد');
606
+ return target.key;
607
+ }
608
+ safeTargets(name, actor, mode) {
609
+ try {
610
+ return this.targets(name, actor, mode === 'upsert' ? 'upsert' : 'create');
294
611
  }
295
612
  catch {
296
613
  return [];
@@ -317,17 +634,33 @@ export class ImportBatches {
317
634
  resourceLabel: label,
318
635
  fileName: String(row.file_name),
319
636
  status: row.status,
637
+ mode: row.mode === 'upsert' ? 'upsert' : 'create',
638
+ matchField: row.match_field ?? null,
639
+ dryRun: Boolean(row.dry_run),
320
640
  headers: row.headers ?? [],
321
641
  mapping: row.mapping ?? {},
322
642
  sample: (row.rows ?? []).slice(0, 5),
323
643
  total: Number(row.total),
324
644
  processed: Number(row.processed ?? 0),
325
645
  created: Number(row.created ?? 0),
646
+ updated: Number(row.updated ?? 0),
326
647
  failed: Number(row.failed ?? 0),
327
648
  errors: row.errors ?? [],
649
+ report: row.report ?? [],
328
650
  createdAt: new Date(row.created_at).toISOString(),
329
651
  finishedAt: row.finished_at ? new Date(row.finished_at).toISOString() : null,
330
652
  targets,
331
653
  };
332
654
  }
333
655
  }
656
+ /** A row failure as the user reads it; database constraint errors get their Arabic text. */
657
+ function rowMessage(error) {
658
+ const code = error?.code;
659
+ if (code === '23505')
660
+ return translate('errors.E_DUPLICATE', 'ar');
661
+ if (code === '23503')
662
+ return translate('errors.E_RELATION', 'ar');
663
+ return error instanceof AdulaError || error instanceof Error
664
+ ? error.message.slice(0, 300)
665
+ : 'خطأ غير معروف';
666
+ }