@navette-js/panels 1.0.0-beta.3 → 1.0.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD-PARTY-NOTICES.md +2 -1
- package/dist/navette/navette/calendar.css +5 -5
- package/dist/navette/navette/motion.css +21 -21
- package/dist/navette/navette/motion.js +6 -6
- package/dist/navette/navette/security-keys.css +11 -0
- package/dist/navette/navette/security-keys.js +72 -0
- package/lib/auth.d.ts +17 -0
- package/lib/auth.js +70 -0
- package/lib/error-pages.d.ts +9 -0
- package/lib/error-pages.js +173 -0
- package/lib/import-export.d.ts +185 -0
- package/lib/import-export.js +543 -0
- package/lib/index.d.ts +629 -79
- package/lib/index.js +1945 -333
- package/lib/multi-factor.d.ts +348 -0
- package/lib/multi-factor.js +928 -0
- package/lib/notifications.d.ts +20 -2
- package/lib/notifications.js +34 -4
- package/lib/qr-code.d.ts +16 -0
- package/lib/qr-code.js +255 -0
- package/lib/session-store.d.ts +21 -0
- package/lib/session-store.js +35 -0
- package/lib/throttle.d.ts +10 -0
- package/lib/throttle.js +25 -0
- package/lib/topbar.d.ts +3 -1
- package/lib/topbar.js +18 -1
- package/lib/totp.d.ts +38 -0
- package/lib/totp.js +104 -0
- package/lib/webauthn.d.ts +155 -0
- package/lib/webauthn.js +287 -0
- package/lib/xlsx.d.ts +7 -0
- package/lib/xlsx.js +171 -0
- package/package.json +9 -9
- package/views/components/tenant-menu.ejs +3 -3
- package/views/components/user-menu.ejs +3 -2
- package/views/errors/page.ejs +87 -0
- package/views/layout/base.ejs +9 -1
- package/views/livewire/sidebar.ejs +35 -20
- package/views/livewire/topbar.ejs +1 -1
- package/views/pages/page.ejs +53 -35
- package/views/pages/simple.ejs +4 -0
|
@@ -0,0 +1,543 @@
|
|
|
1
|
+
// Import and export actions: CSV and XLSX files, read and written during the request or in the background (queued()),
|
|
2
|
+
// an optional history of runs, and a per-user download store.
|
|
3
|
+
import { buildXlsx, readXlsx } from "./xlsx.js";
|
|
4
|
+
import { randomBytes } from 'node:crypto';
|
|
5
|
+
import { readFileSync, statSync } from 'node:fs';
|
|
6
|
+
import { Action } from '@navette-js/actions';
|
|
7
|
+
import { CheckboxList, FileUpload, Select } from '@navette-js/forms';
|
|
8
|
+
import { Fieldset } from '@navette-js/schemas';
|
|
9
|
+
import { configured } from '@navette-js/support';
|
|
10
|
+
import { t, tc } from '@navette-js/support/i18n';
|
|
11
|
+
import { getTableQuery } from '@navette-js/tables';
|
|
12
|
+
import { Notification } from "./notifications.js";
|
|
13
|
+
/**
|
|
14
|
+
* Files offered for download to one user (exports, failed import rows), kept in memory for a limited time. They are lost
|
|
15
|
+
* when the process restarts and are not shared between processes.
|
|
16
|
+
*/
|
|
17
|
+
export class DownloadStore {
|
|
18
|
+
files = new Map();
|
|
19
|
+
ttl;
|
|
20
|
+
maxFiles;
|
|
21
|
+
constructor(options = {}) { this.ttl = options.ttl ?? 24 * 60 * 60 * 1000; this.maxFiles = options.maxFiles ?? 500; }
|
|
22
|
+
/** Stores a file for a user and returns its token (part of the download URL). */
|
|
23
|
+
put(owner, name, type, content) {
|
|
24
|
+
const now = Date.now();
|
|
25
|
+
for (const [token, file] of this.files)
|
|
26
|
+
if (file.expires < now)
|
|
27
|
+
this.files.delete(token);
|
|
28
|
+
while (this.files.size >= this.maxFiles)
|
|
29
|
+
this.files.delete(this.files.keys().next().value);
|
|
30
|
+
const token = randomBytes(24).toString('base64url');
|
|
31
|
+
this.files.set(token, { owner: String(owner), name, type, content: Buffer.isBuffer(content) ? content : Buffer.from(content, 'utf8'), expires: now + this.ttl });
|
|
32
|
+
return token;
|
|
33
|
+
}
|
|
34
|
+
/** The file of a token, only for the user who created it. */
|
|
35
|
+
get(token, owner) {
|
|
36
|
+
const file = this.files.get(token);
|
|
37
|
+
if (!file || file.expires < Date.now() || file.owner !== String(owner))
|
|
38
|
+
return null;
|
|
39
|
+
return file;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
// ------------------------------------------------------------------ CSV
|
|
43
|
+
/**
|
|
44
|
+
* A CSV cell. Text starting with `=`, `+`, `-`, `@`, a tab or a carriage return is prefixed with `'`, so that spreadsheet
|
|
45
|
+
* software does not run it as a formula (CSV injection).
|
|
46
|
+
*/
|
|
47
|
+
export function csvCell(value) {
|
|
48
|
+
let text = value === null || value === undefined ? '' : value instanceof Date ? value.toISOString() : String(value);
|
|
49
|
+
if (/^[=+\-@\t\r]/.test(text) && !/^-?\d+(\.\d+)?$/.test(text))
|
|
50
|
+
text = `'${text}`;
|
|
51
|
+
return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
|
|
52
|
+
}
|
|
53
|
+
export const csvLine = (cells) => cells.map(csvCell).join(',');
|
|
54
|
+
/** Parses CSV text (quoted cells, doubled quotes, line breaks inside quotes). */
|
|
55
|
+
export function parseCsv(text, maxRows = Infinity) {
|
|
56
|
+
const rows = [];
|
|
57
|
+
let row = [], cell = '', quoted = false;
|
|
58
|
+
const source = text.replace(/^/, '');
|
|
59
|
+
for (let index = 0; index < source.length; index++) {
|
|
60
|
+
const char = source[index];
|
|
61
|
+
if (quoted) {
|
|
62
|
+
if (char === '"' && source[index + 1] === '"') {
|
|
63
|
+
cell += '"';
|
|
64
|
+
index++;
|
|
65
|
+
}
|
|
66
|
+
else if (char === '"')
|
|
67
|
+
quoted = false;
|
|
68
|
+
else
|
|
69
|
+
cell += char;
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
if (char === '"')
|
|
73
|
+
quoted = true;
|
|
74
|
+
else if (char === ',') {
|
|
75
|
+
row.push(cell);
|
|
76
|
+
cell = '';
|
|
77
|
+
}
|
|
78
|
+
else if (char === '\n' || char === '\r') {
|
|
79
|
+
if (char === '\r' && source[index + 1] === '\n')
|
|
80
|
+
index++;
|
|
81
|
+
row.push(cell);
|
|
82
|
+
cell = '';
|
|
83
|
+
if (row.some((value) => value !== ''))
|
|
84
|
+
rows.push(row);
|
|
85
|
+
row = [];
|
|
86
|
+
if (rows.length > maxRows)
|
|
87
|
+
return rows;
|
|
88
|
+
}
|
|
89
|
+
else
|
|
90
|
+
cell += char;
|
|
91
|
+
}
|
|
92
|
+
row.push(cell);
|
|
93
|
+
if (row.some((value) => value !== ''))
|
|
94
|
+
rows.push(row);
|
|
95
|
+
return rows;
|
|
96
|
+
}
|
|
97
|
+
// ------------------------------------------------------------------ export
|
|
98
|
+
/** Column of an export: its header and the value written for each record. */
|
|
99
|
+
export class ExportColumn {
|
|
100
|
+
name;
|
|
101
|
+
_label = null;
|
|
102
|
+
_state = null;
|
|
103
|
+
_format = null;
|
|
104
|
+
_enabledByDefault = true;
|
|
105
|
+
constructor(name) { this.name = name; }
|
|
106
|
+
static make(name) { return configured(new ExportColumn(name)); }
|
|
107
|
+
label(label) { this._label = label; return this; }
|
|
108
|
+
getLabel() { return this._label ?? this.name.replace(/[_.]+/g, ' ').replace(/^./, (c) => c.toUpperCase()); }
|
|
109
|
+
/** Value from the record, instead of the attribute with the column's name (dots read relationships: `author.name`). */
|
|
110
|
+
state(resolver) { this._state = resolver; return this; }
|
|
111
|
+
formatStateUsing(format) { this._format = format; return this; }
|
|
112
|
+
/** `false`: unchecked in the export modal. */
|
|
113
|
+
enabledByDefault(condition = true) { this._enabledByDefault = condition; return this; }
|
|
114
|
+
isEnabledByDefault() { return this._enabledByDefault; }
|
|
115
|
+
getValue(record) {
|
|
116
|
+
const state = this._state ? this._state(record) : this.name.split('.').reduce((value, key) => (value === null || value === undefined ? value : value[key]), record);
|
|
117
|
+
return this._format ? this._format(state, record) : state;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
/** Describes an export: its columns, and optionally the query and the notification. */
|
|
121
|
+
export class Exporter {
|
|
122
|
+
static getColumns() { return []; }
|
|
123
|
+
/** Changes the query of the exported records (e.g. eager loading). */
|
|
124
|
+
static modifyQuery(_query) { }
|
|
125
|
+
/** Body of the "Export completed" notification. */
|
|
126
|
+
static getCompletedNotificationBody(count) { return `Your export has completed and ${count.toLocaleString('en-US')} ${count === 1 ? 'row' : 'rows'} exported.`; }
|
|
127
|
+
/** File name, without extension. */
|
|
128
|
+
static getFileName(_count) { return `export-${new Date().toISOString().slice(0, 10)}`; }
|
|
129
|
+
}
|
|
130
|
+
const DEFAULT_MAX_EXPORT_ROWS = 10000, HARD_MAX_EXPORT_ROWS = 100000, CHUNK = 1000;
|
|
131
|
+
const fileNameOf = (name) => name.replace(/[^A-Za-z0-9._-]+/g, '-').slice(0, 120) || 'export';
|
|
132
|
+
function makeExportAction(name, bulk) {
|
|
133
|
+
let exporter = null, maxRows = DEFAULT_MAX_EXPORT_ROWS, mapping = true, formats = ['csv', 'xlsx'], queued = false;
|
|
134
|
+
const columnsOf = () => { if (!exporter)
|
|
135
|
+
throw new Error('ExportAction: call exporter() with an Exporter class.'); return exporter.getColumns(); };
|
|
136
|
+
const action = Action.make(name).color('gray').icon('m-arrow-down-tray')
|
|
137
|
+
// Exporting reads the records: the resource's `viewAny` ability (or the host's own checks) applies.
|
|
138
|
+
.authorize(async ({ livewire }) => (typeof livewire?.resource?.can === 'function' ? livewire.resource.can('viewAny', livewire._ctx?.user ?? null) : true))
|
|
139
|
+
.label(({ livewire }) => t('filament-actions::export.label', { label: livewire?.resource?.getPluralModelLabel?.() ?? '' }).trim())
|
|
140
|
+
.modalHeading(({ livewire }) => t('filament-actions::export.modal.heading', { label: livewire?.resource?.getPluralModelLabel?.() ?? '' }).trim())
|
|
141
|
+
.modalSubmitActionLabel(t('filament-actions::export.modal.actions.export.label'))
|
|
142
|
+
.modalWidth('md')
|
|
143
|
+
.schema(() => (mapping ? [CheckboxList.make('columns').label(t('filament-actions::export.modal.form.columns.label')).options(columnsOf().map((column) => [column.name, column.getLabel()])).columns(2)] : []))
|
|
144
|
+
.fillForm(() => ({ columns: columnsOf().filter((column) => column.isEnabledByDefault()).map((column) => column.name) }))
|
|
145
|
+
.action(async ({ livewire, data }) => {
|
|
146
|
+
// Only declared columns are exported, whatever list the browser sends.
|
|
147
|
+
const requested = mapping ? (Array.isArray(data.columns) ? data.columns.map(String) : []) : columnsOf().map((column) => column.name);
|
|
148
|
+
const columns = columnsOf().filter((column) => requested.includes(column.name));
|
|
149
|
+
if (!columns.length) {
|
|
150
|
+
Notification.make().danger().title(t('filament-actions::export.notifications.no_columns.title')).body(t('filament-actions::export.notifications.no_columns.body')).send();
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
const adapter = livewire.getDataAdapterForTable();
|
|
154
|
+
const table = livewire.getTable();
|
|
155
|
+
// The table's current query (filters, search, sort); a bulk export only takes the selected records.
|
|
156
|
+
const query = { ...getTableQuery(table, livewire), modifiers: [...(getTableQuery(table, livewire).modifiers ?? []), (q) => exporter.modifyQuery(q)] };
|
|
157
|
+
if (bulk)
|
|
158
|
+
query.keys = await livewire.getSelectedRecordKeys();
|
|
159
|
+
const first = await adapter.paginate({ ...query, page: 1, perPage: CHUNK });
|
|
160
|
+
const limit = Math.min(maxRows, HARD_MAX_EXPORT_ROWS);
|
|
161
|
+
if (first.total > limit) {
|
|
162
|
+
Notification.make().danger().title(t('filament-actions::export.notifications.max_rows.title')).body(tc('filament-actions::export.notifications.max_rows.body', limit, { count: limit.toLocaleString('en-US') })).send();
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
const user = livewire._ctx?.user;
|
|
166
|
+
const panel = livewire._ctx?.panel;
|
|
167
|
+
const background = queued && canQueue(panel);
|
|
168
|
+
const history = panel?.getImportExportHistory?.() ?? null;
|
|
169
|
+
const run = async () => {
|
|
170
|
+
const exportId = history ? await history.startExport({ userId: user.id, exporter: exporter.name, fileName: fileNameOf(exporter.getFileName(first.total)), totalRows: first.total }) : null;
|
|
171
|
+
const rows = [columns.map((column) => column.getLabel())];
|
|
172
|
+
for (let page = 1; page <= first.lastPage; page++) {
|
|
173
|
+
const { records } = page === 1 ? first : await adapter.paginate({ ...query, page, perPage: CHUNK });
|
|
174
|
+
for (const record of records)
|
|
175
|
+
rows.push(columns.map((column) => column.getValue(record)));
|
|
176
|
+
}
|
|
177
|
+
const count = rows.length - 1;
|
|
178
|
+
const fileName = fileNameOf(exporter.getFileName(count));
|
|
179
|
+
const links = [];
|
|
180
|
+
if (formats.includes('csv'))
|
|
181
|
+
links.push({ label: t('filament-actions::export.notifications.completed.actions.download_csv.label'), url: panel.getDownloadUrl(panel.getDownloads().put(user.id, `${fileName}.csv`, 'text/csv; charset=utf-8', `${rows.map(csvLine).join('\r\n')}\r\n`)) });
|
|
182
|
+
if (formats.includes('xlsx'))
|
|
183
|
+
links.push({ label: t('filament-actions::export.notifications.completed.actions.download_xlsx.label'), url: panel.getDownloadUrl(panel.getDownloads().put(user.id, `${fileName}.xlsx`, 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', buildXlsx(rows))) });
|
|
184
|
+
if (history && exportId !== null)
|
|
185
|
+
await history.finishExport(exportId, { successfulRows: count });
|
|
186
|
+
await deliver(Notification.make().success().persistent().title(t('filament-actions::export.notifications.completed.title')).body(exporter.getCompletedNotificationBody(count)).actions(links), background, panel, user);
|
|
187
|
+
};
|
|
188
|
+
if (bulk)
|
|
189
|
+
livewire.deselectAllTableRecords();
|
|
190
|
+
if (!background) {
|
|
191
|
+
await run();
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
Notification.make().success().title(t('filament-actions::export.notifications.started.title')).body(tc('filament-actions::export.notifications.started.body', first.total, { count: first.total.toLocaleString('en-US') })).send();
|
|
195
|
+
runInBackground(run);
|
|
196
|
+
});
|
|
197
|
+
action.exporter = (value) => { exporter = value; return action; };
|
|
198
|
+
action.maxRows = (count) => { maxRows = Math.max(1, Math.floor(count)); return action; };
|
|
199
|
+
action.formats = (list) => { const allowed = list.filter((format) => format === 'csv' || format === 'xlsx'); if (!allowed.length)
|
|
200
|
+
throw new Error('ExportAction: formats() needs csv and/or xlsx.'); formats = [...new Set(allowed)]; return action; };
|
|
201
|
+
action.columnMapping = (condition) => { mapping = condition; if (!condition)
|
|
202
|
+
action.requiresConfirmation(); return action; };
|
|
203
|
+
action.queued = (condition = true) => { queued = condition; return action; };
|
|
204
|
+
return action;
|
|
205
|
+
}
|
|
206
|
+
/** Exports the records of the table (with its filters, search and sort) to a CSV file. */
|
|
207
|
+
export class ExportAction {
|
|
208
|
+
static make(name = 'export') { return makeExportAction(name, false); }
|
|
209
|
+
}
|
|
210
|
+
/** Exports the selected records to a CSV file. */
|
|
211
|
+
export class ExportBulkAction {
|
|
212
|
+
static make(name = 'export') { return makeExportAction(name, true); }
|
|
213
|
+
}
|
|
214
|
+
/** Column of an import: the file column it is read from, its validation and how the value is cast. */
|
|
215
|
+
export class ImportColumn {
|
|
216
|
+
name;
|
|
217
|
+
_label = null;
|
|
218
|
+
_guesses = [];
|
|
219
|
+
_required = false;
|
|
220
|
+
_rules = [];
|
|
221
|
+
_cast = null;
|
|
222
|
+
_example = null;
|
|
223
|
+
_fill = null;
|
|
224
|
+
constructor(name) { this.name = name; }
|
|
225
|
+
static make(name) { return configured(new ImportColumn(name)); }
|
|
226
|
+
label(label) { this._label = label; return this; }
|
|
227
|
+
getLabel() { return this._label ?? this.name.replace(/[_.]+/g, ' ').replace(/^./, (c) => c.toUpperCase()); }
|
|
228
|
+
/** Other file headers that map to this column (compared case-insensitively). */
|
|
229
|
+
guess(headers) { this._guesses = headers; return this; }
|
|
230
|
+
/** The file must contain this column. */
|
|
231
|
+
requiredMapping(condition = true) { this._required = condition; return this; }
|
|
232
|
+
isMappingRequired() { return this._required; }
|
|
233
|
+
/** `required`, `email`, `numeric`, `integer`, `boolean`, `min:n`, `max:n`, `in:a,b`, or a function returning an error message. */
|
|
234
|
+
rules(rules) { this._rules = rules; return this; }
|
|
235
|
+
/** Converts the text of the cell before it is saved. */
|
|
236
|
+
castStateUsing(cast) { this._cast = cast; return this; }
|
|
237
|
+
numeric() { return this.castStateUsing((value) => (value === null || value === '' ? null : Number(value))); }
|
|
238
|
+
boolean() { return this.castStateUsing((value) => (value === null || value === '' ? null : ['1', 'true', 'yes', 'y', 'on'].includes(value.toLowerCase()) ? 1 : 0)); }
|
|
239
|
+
/** Value written in the example file. */
|
|
240
|
+
example(value) { this._example = value; return this; }
|
|
241
|
+
getExample() { return this._example ?? ''; }
|
|
242
|
+
/** Writes the value into the record yourself, instead of the attribute with the column's name. */
|
|
243
|
+
fillRecordUsing(fill) { this._fill = fill; return this; }
|
|
244
|
+
matches(header) { const normalized = header.trim().toLowerCase(); return [this.name, this.getLabel(), ...this._guesses].some((candidate) => candidate.toLowerCase() === normalized); }
|
|
245
|
+
validate(value, row) {
|
|
246
|
+
const label = this.getLabel().toLowerCase();
|
|
247
|
+
const blank = value === null || value.trim() === '';
|
|
248
|
+
for (const rule of this._rules) {
|
|
249
|
+
if (typeof rule === 'function') {
|
|
250
|
+
const message = rule(value, row);
|
|
251
|
+
if (message)
|
|
252
|
+
return message;
|
|
253
|
+
continue;
|
|
254
|
+
}
|
|
255
|
+
const [name, parameter = ''] = rule.split(':');
|
|
256
|
+
if (name === 'required' && blank)
|
|
257
|
+
return `The ${label} field is required.`;
|
|
258
|
+
if (blank)
|
|
259
|
+
continue;
|
|
260
|
+
const text = value.trim();
|
|
261
|
+
if (name === 'email' && !/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(text))
|
|
262
|
+
return `The ${label} field must be a valid email address.`;
|
|
263
|
+
if (name === 'numeric' && !Number.isFinite(Number(text)))
|
|
264
|
+
return `The ${label} field must be a number.`;
|
|
265
|
+
if (name === 'integer' && !/^-?\d+$/.test(text))
|
|
266
|
+
return `The ${label} field must be an integer.`;
|
|
267
|
+
if (name === 'boolean' && !['0', '1', 'true', 'false', 'yes', 'no'].includes(text.toLowerCase()))
|
|
268
|
+
return `The ${label} field must be true or false.`;
|
|
269
|
+
if (name === 'max' && text.length > Number(parameter))
|
|
270
|
+
return `The ${label} field must not be greater than ${parameter} characters.`;
|
|
271
|
+
if (name === 'min' && text.length < Number(parameter))
|
|
272
|
+
return `The ${label} field must be at least ${parameter} characters.`;
|
|
273
|
+
if (name === 'in' && !parameter.split(',').includes(text))
|
|
274
|
+
return `The selected ${label} is invalid.`;
|
|
275
|
+
}
|
|
276
|
+
return null;
|
|
277
|
+
}
|
|
278
|
+
cast(value) { return this._cast ? this._cast(value) : value === '' ? null : value; }
|
|
279
|
+
fill(record, value) { if (this._fill)
|
|
280
|
+
this._fill(record, value);
|
|
281
|
+
else
|
|
282
|
+
record[this.name] = value; }
|
|
283
|
+
}
|
|
284
|
+
/** Describes an import: its columns, how a row finds or creates its record, and hooks. */
|
|
285
|
+
export class Importer {
|
|
286
|
+
static getColumns() { return []; }
|
|
287
|
+
/**
|
|
288
|
+
* Existing record a row updates, or `null` to create one (the default). For example, find a product by its SKU with the
|
|
289
|
+
* data adapter. Only return records the user may change.
|
|
290
|
+
*/
|
|
291
|
+
static async resolveRecord(_data, _ctx) { return null; }
|
|
292
|
+
/** Runs before a record is saved; `data` may be changed. */
|
|
293
|
+
static async beforeSave(_data, _record) { }
|
|
294
|
+
static async afterSave(_record) { }
|
|
295
|
+
static getCompletedNotificationBody(imported, failed) {
|
|
296
|
+
return `Your import has completed and ${imported.toLocaleString('en-US')} ${imported === 1 ? 'row' : 'rows'} imported.${failed ? ` ${failed.toLocaleString('en-US')} ${failed === 1 ? 'row' : 'rows'} failed to import.` : ''}`;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
const DEFAULT_MAX_IMPORT_ROWS = 1000, HARD_MAX_IMPORT_ROWS = 10000, HARD_MAX_QUEUED_IMPORT_ROWS = 100000;
|
|
300
|
+
/** Rows of an uploaded CSV or XLSX file (detected from its content, never its name), up to `limit` rows. */
|
|
301
|
+
export function readSpreadsheet(path, limit) {
|
|
302
|
+
const buffer = readFileSync(path);
|
|
303
|
+
if (buffer.length >= 4 && buffer.readUInt32LE(0) === 0x04034b50)
|
|
304
|
+
return readXlsx(buffer, limit);
|
|
305
|
+
return parseCsv(buffer.toString('utf8').replace(/^/, ''), limit);
|
|
306
|
+
}
|
|
307
|
+
/** Path of the temporary file uploaded in a FileUpload state, if any. */
|
|
308
|
+
function temporaryPathOf(livewire, state) {
|
|
309
|
+
// A saved entry (`{ file: TemporaryUpload }`), a list of them, or the raw state of the field (upload references).
|
|
310
|
+
if (state && typeof state === 'object' && typeof state.file?.path === 'string')
|
|
311
|
+
return state.file.path;
|
|
312
|
+
const values = Array.isArray(state) ? state : state && typeof state === 'object' ? Object.values(state) : [state];
|
|
313
|
+
for (const value of values) {
|
|
314
|
+
const file = (value && typeof value === 'object' && value.file?.path) ? value.file : livewire?._runtime?.uploads?.resolve?.(value);
|
|
315
|
+
if (typeof file?.path === 'string')
|
|
316
|
+
return file.path;
|
|
317
|
+
}
|
|
318
|
+
return null;
|
|
319
|
+
}
|
|
320
|
+
const headersOf = (livewire, state) => {
|
|
321
|
+
const path = temporaryPathOf(livewire, state);
|
|
322
|
+
if (!path)
|
|
323
|
+
return [];
|
|
324
|
+
try {
|
|
325
|
+
return (readSpreadsheet(path, 1)[0] ?? []).map((cell) => cell.trim()).filter((cell, index, all) => cell !== '' && all.indexOf(cell) === index);
|
|
326
|
+
}
|
|
327
|
+
catch {
|
|
328
|
+
return [];
|
|
329
|
+
}
|
|
330
|
+
};
|
|
331
|
+
const pendingTasks = [];
|
|
332
|
+
let draining = false;
|
|
333
|
+
async function drainTasks() {
|
|
334
|
+
if (draining)
|
|
335
|
+
return;
|
|
336
|
+
draining = true;
|
|
337
|
+
while (pendingTasks.length) {
|
|
338
|
+
const task = pendingTasks.shift();
|
|
339
|
+
try {
|
|
340
|
+
await task();
|
|
341
|
+
}
|
|
342
|
+
catch (error) {
|
|
343
|
+
console.error('[navette] background task failed:', error);
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
draining = false;
|
|
347
|
+
}
|
|
348
|
+
let backgroundRunner = (task) => { pendingTasks.push(task); setImmediate(() => { void drainTasks(); }); };
|
|
349
|
+
/**
|
|
350
|
+
* Runs queued imports and exports. By default, one at a time in this process, after the response. Plug your own queue (BullMQ,
|
|
351
|
+
* a worker…) with a function receiving each task.
|
|
352
|
+
*/
|
|
353
|
+
export const useBackgroundRunner = (runner) => { backgroundRunner = runner; };
|
|
354
|
+
export const runInBackground = (task) => backgroundRunner(task);
|
|
355
|
+
const now = () => new Date().toISOString().slice(0, 19).replace('T', ' ');
|
|
356
|
+
/** History in `imports`, `exports` and `failed_import_rows` tables (create them with `importExportSchema()`). */
|
|
357
|
+
export function knexImportExportHistory(db) {
|
|
358
|
+
const insert = async (table, values) => { const [id] = await db(table).insert(values); return typeof id === 'object' && id !== null ? id.id : id; };
|
|
359
|
+
return {
|
|
360
|
+
startImport: (data) => insert('imports', { user_id: data.userId, importer: data.importer, file_name: data.fileName.slice(0, 255), total_rows: data.totalRows, processed_rows: 0, successful_rows: 0, created_at: now(), updated_at: now() }),
|
|
361
|
+
async finishImport(id, data) { await db('imports').where('id', id).update({ processed_rows: data.processedRows, successful_rows: data.successfulRows, completed_at: now(), updated_at: now() }); },
|
|
362
|
+
async recordFailedImportRow(importId, row, error) { await db('failed_import_rows').insert({ import_id: importId, data: JSON.stringify(row), validation_error: error.slice(0, 1000), created_at: now(), updated_at: now() }); },
|
|
363
|
+
startExport: (data) => insert('exports', { user_id: data.userId, exporter: data.exporter, file_name: data.fileName.slice(0, 255), total_rows: data.totalRows, successful_rows: 0, created_at: now(), updated_at: now() }),
|
|
364
|
+
async finishExport(id, data) { await db('exports').where('id', id).update({ successful_rows: data.successfulRows, completed_at: now(), updated_at: now() }); },
|
|
365
|
+
};
|
|
366
|
+
}
|
|
367
|
+
/** Creates the history tables when missing (call it from a migration). */
|
|
368
|
+
export async function importExportSchema(knex) {
|
|
369
|
+
if (!(await knex.schema.hasTable('imports')))
|
|
370
|
+
await knex.schema.createTable('imports', (table) => {
|
|
371
|
+
table.increments('id');
|
|
372
|
+
table.integer('user_id').index();
|
|
373
|
+
table.string('importer');
|
|
374
|
+
table.string('file_name');
|
|
375
|
+
table.integer('total_rows');
|
|
376
|
+
table.integer('processed_rows').defaultTo(0);
|
|
377
|
+
table.integer('successful_rows').defaultTo(0);
|
|
378
|
+
table.timestamp('completed_at').nullable();
|
|
379
|
+
table.timestamps();
|
|
380
|
+
});
|
|
381
|
+
if (!(await knex.schema.hasTable('failed_import_rows')))
|
|
382
|
+
await knex.schema.createTable('failed_import_rows', (table) => {
|
|
383
|
+
table.increments('id');
|
|
384
|
+
table.integer('import_id').index();
|
|
385
|
+
table.text('data');
|
|
386
|
+
table.text('validation_error').nullable();
|
|
387
|
+
table.timestamps();
|
|
388
|
+
});
|
|
389
|
+
if (!(await knex.schema.hasTable('exports')))
|
|
390
|
+
await knex.schema.createTable('exports', (table) => {
|
|
391
|
+
table.increments('id');
|
|
392
|
+
table.integer('user_id').index();
|
|
393
|
+
table.string('exporter');
|
|
394
|
+
table.string('file_name');
|
|
395
|
+
table.integer('total_rows');
|
|
396
|
+
table.integer('successful_rows').defaultTo(0);
|
|
397
|
+
table.timestamp('completed_at').nullable();
|
|
398
|
+
table.timestamps();
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
/** Sends a notification now (in the request) or, for background work, to the user's database notifications. */
|
|
402
|
+
async function deliver(notification, background, panel, user) {
|
|
403
|
+
if (background)
|
|
404
|
+
await notification.sendToDatabase(panel.getDatabaseNotifications?.() ?? null, user.id);
|
|
405
|
+
else
|
|
406
|
+
notification.send();
|
|
407
|
+
}
|
|
408
|
+
/** Background work needs database notifications to tell the user it ended; without them, it runs in the request. */
|
|
409
|
+
const canQueue = (panel) => Boolean(panel?.getDatabaseNotifications?.());
|
|
410
|
+
/** Imports the rows of a CSV or XLSX file: each row is validated, then creates or updates a record through the data adapter. */
|
|
411
|
+
export class ImportAction {
|
|
412
|
+
static make(name = 'import') {
|
|
413
|
+
let importer = null, maxRows = DEFAULT_MAX_IMPORT_ROWS, maxKilobytes = 5120, mapping = true, queued = false;
|
|
414
|
+
const columnsOf = () => { if (!importer)
|
|
415
|
+
throw new Error('ImportAction: call importer() with an Importer class.'); return importer.getColumns(); };
|
|
416
|
+
const example = Action.make('downloadExample').label(t('filament-actions::import.modal.actions.download_example.label')).color('gray').link()
|
|
417
|
+
.action(({ livewire }) => {
|
|
418
|
+
const columns = columnsOf();
|
|
419
|
+
const csv = `${csvLine(columns.map((column) => column.name))}\r\n${csvLine(columns.map((column) => column.getExample()))}\r\n`;
|
|
420
|
+
const panel = livewire._ctx?.panel;
|
|
421
|
+
livewire.redirect(panel.getDownloadUrl(panel.getDownloads().put(livewire._ctx.user.id, 'import-example.csv', 'text/csv; charset=utf-8', csv)));
|
|
422
|
+
});
|
|
423
|
+
const action = Action.make(name).color('gray').icon('m-arrow-up-tray')
|
|
424
|
+
// Importing creates or updates records: the resource's `create` ability applies.
|
|
425
|
+
.authorize(async ({ livewire }) => (typeof livewire?.resource?.can === 'function' ? livewire.resource.can('create', livewire._ctx?.user ?? null) : true))
|
|
426
|
+
.label(({ livewire }) => t('filament-actions::import.label', { label: livewire?.resource?.getPluralModelLabel?.() ?? '' }).trim())
|
|
427
|
+
.modalHeading(({ livewire }) => t('filament-actions::import.modal.heading', { label: livewire?.resource?.getPluralModelLabel?.() ?? '' }).trim())
|
|
428
|
+
.modalSubmitActionLabel(t('filament-actions::import.modal.actions.import.label'))
|
|
429
|
+
.modalWidth('md')
|
|
430
|
+
.schema((schema) => {
|
|
431
|
+
const livewire = schema.livewire;
|
|
432
|
+
const columns = columnsOf();
|
|
433
|
+
// The columns of the uploaded file, to map them to the importer's columns.
|
|
434
|
+
const mounted = livewire?.mountedActions?.find?.((candidate) => candidate.name === name);
|
|
435
|
+
const headers = mapping ? headersOf(livewire, mounted?.data?.file) : [];
|
|
436
|
+
const file = FileUpload.make('file').label(t('filament-actions::import.modal.form.file.label')).placeholder(t('filament-actions::import.modal.form.file.placeholder'))
|
|
437
|
+
.acceptedFileTypes(['text/csv', 'text/plain', 'application/vnd.ms-excel', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', '.csv', '.xlsx']).maxSize(maxKilobytes).storeFiles(false).required();
|
|
438
|
+
if (mapping)
|
|
439
|
+
file.live().afterStateUpdated(({ state, set }) => {
|
|
440
|
+
const found = headersOf(livewire, state);
|
|
441
|
+
set('columnMap', Object.fromEntries(columns.map((column) => [column.name, found.find((header) => column.matches(header)) ?? null])));
|
|
442
|
+
});
|
|
443
|
+
return schema.components([
|
|
444
|
+
file,
|
|
445
|
+
...(mapping && headers.length ? [Fieldset.make(t('filament-actions::import.modal.form.columns.label')).statePath('columnMap').columns(1).schema(columns.map((column) => Select.make(column.name).label(column.getLabel()).placeholder(t('filament-actions::import.modal.form.columns.placeholder')).options(headers.map((header) => [header, header])).required(column.isMappingRequired())))] : []),
|
|
446
|
+
]);
|
|
447
|
+
})
|
|
448
|
+
.extraModalFooterActions([example])
|
|
449
|
+
.action(async ({ livewire, data }) => {
|
|
450
|
+
const path = temporaryPathOf(livewire, data.file);
|
|
451
|
+
// Only a temporary upload of this request is read, never a path sent by the browser.
|
|
452
|
+
if (typeof path !== 'string' || statSync(path).size > maxKilobytes * 1024)
|
|
453
|
+
return;
|
|
454
|
+
const panel = livewire._ctx?.panel, user = livewire._ctx?.user;
|
|
455
|
+
const background = queued && canQueue(panel);
|
|
456
|
+
const limit = Math.min(maxRows, background ? HARD_MAX_QUEUED_IMPORT_ROWS : HARD_MAX_IMPORT_ROWS);
|
|
457
|
+
let rows;
|
|
458
|
+
try {
|
|
459
|
+
rows = readSpreadsheet(path, limit + 1);
|
|
460
|
+
}
|
|
461
|
+
catch {
|
|
462
|
+
Notification.make().danger().title(t('filament-actions::import.notifications.completed.title')).body('The file could not be read: upload a CSV or XLSX file.').send();
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
const header = (rows.shift() ?? []).map((cell) => cell.trim());
|
|
466
|
+
if (rows.length > limit) {
|
|
467
|
+
Notification.make().danger().title(t('filament-actions::import.notifications.max_rows.title')).body(tc('filament-actions::import.notifications.max_rows.body', limit, { count: limit.toLocaleString('en-US') })).send();
|
|
468
|
+
return;
|
|
469
|
+
}
|
|
470
|
+
const columns = columnsOf();
|
|
471
|
+
// Chosen columns (only headers of the file are accepted), else automatic matching.
|
|
472
|
+
const chosen = (data.columnMap && typeof data.columnMap === 'object' ? data.columnMap : {});
|
|
473
|
+
const indexOf = (column) => {
|
|
474
|
+
if (mapping && Object.hasOwn(chosen, column.name))
|
|
475
|
+
return typeof chosen[column.name] === 'string' ? header.indexOf(chosen[column.name]) : -1;
|
|
476
|
+
return header.findIndex((cell) => column.matches(cell));
|
|
477
|
+
};
|
|
478
|
+
const columnIndexes = new Map(columns.map((column) => [column, indexOf(column)]));
|
|
479
|
+
const missing = columns.filter((column) => column.isMappingRequired() && columnIndexes.get(column) === -1);
|
|
480
|
+
if (missing.length) {
|
|
481
|
+
Notification.make().danger().title(t('filament-actions::import.notifications.completed.title'))
|
|
482
|
+
.body(missing.map((column) => t('filament-actions::import.failure_csv.column_mapping_required_for_new_record', { attribute: column.getLabel() })).join(' ')).send();
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
const adapter = livewire.getDataAdapterForTable();
|
|
486
|
+
const history = panel?.getImportExportHistory?.() ?? null;
|
|
487
|
+
const fileName = String(Object.values(data.file ?? {})[0]?.file?.name ?? Object.values(data.file ?? {})[0]?.name ?? 'import');
|
|
488
|
+
const run = async () => {
|
|
489
|
+
const importId = history ? await history.startImport({ userId: user.id, importer: importer.name, fileName, totalRows: rows.length }) : null;
|
|
490
|
+
let imported = 0;
|
|
491
|
+
const failures = [];
|
|
492
|
+
for (const cells of rows) {
|
|
493
|
+
const raw = {};
|
|
494
|
+
for (const [column, index] of columnIndexes)
|
|
495
|
+
raw[column.name] = index === -1 ? null : (cells[index] ?? null);
|
|
496
|
+
const fail = async (message) => { failures.push(csvLine([...cells, message])); if (history && importId !== null)
|
|
497
|
+
await history.recordFailedImportRow(importId, raw, message); };
|
|
498
|
+
const error = columns.map((column) => (columnIndexes.get(column) === -1 ? null : column.validate(raw[column.name], raw))).find((message) => message !== null);
|
|
499
|
+
if (error) {
|
|
500
|
+
await fail(error);
|
|
501
|
+
continue;
|
|
502
|
+
}
|
|
503
|
+
try {
|
|
504
|
+
const record = await importer.resolveRecord(raw, { livewire });
|
|
505
|
+
const values = {};
|
|
506
|
+
for (const column of columns)
|
|
507
|
+
if (columnIndexes.get(column) !== -1)
|
|
508
|
+
column.fill(values, column.cast(raw[column.name]));
|
|
509
|
+
await importer.beforeSave(values, record);
|
|
510
|
+
const saved = record ? await adapter.update(adapter.getKey(record), values) : await adapter.create(values);
|
|
511
|
+
await importer.afterSave(saved);
|
|
512
|
+
imported++;
|
|
513
|
+
}
|
|
514
|
+
catch (failure) {
|
|
515
|
+
console.error(failure);
|
|
516
|
+
await fail(t('filament-actions::import.failure_csv.system_error'));
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
if (history && importId !== null)
|
|
520
|
+
await history.finishImport(importId, { processedRows: rows.length, successfulRows: imported });
|
|
521
|
+
const notification = Notification.make().title(t('filament-actions::import.notifications.completed.title')).body(importer.getCompletedNotificationBody(imported, failures.length)).persistent();
|
|
522
|
+
if (failures.length) {
|
|
523
|
+
const csv = `${[csvLine([...header, t('filament-actions::import.failure_csv.error_header')]), ...failures].join('\r\n')}\r\n`;
|
|
524
|
+
const token = panel.getDownloads().put(user.id, 'import-failed-rows.csv', 'text/csv; charset=utf-8', csv);
|
|
525
|
+
notification.actions([{ label: tc('filament-actions::import.notifications.completed.actions.download_failed_rows_csv.label', failures.length), url: panel.getDownloadUrl(token) }]);
|
|
526
|
+
}
|
|
527
|
+
await deliver(imported ? (failures.length ? notification.warning() : notification.success()) : notification.danger(), background, panel, user);
|
|
528
|
+
};
|
|
529
|
+
if (!background) {
|
|
530
|
+
await run();
|
|
531
|
+
return;
|
|
532
|
+
}
|
|
533
|
+
Notification.make().success().title(t('filament-actions::import.notifications.started.title')).body(tc('filament-actions::import.notifications.started.body', rows.length, { count: rows.length.toLocaleString('en-US') })).send();
|
|
534
|
+
runInBackground(run);
|
|
535
|
+
});
|
|
536
|
+
action.importer = (value) => { importer = value; return action; };
|
|
537
|
+
action.maxRows = (count) => { maxRows = Math.max(1, Math.floor(count)); return action; };
|
|
538
|
+
action.maxFileSize = (kilobytes) => { maxKilobytes = Math.max(1, Math.floor(kilobytes)); return action; };
|
|
539
|
+
action.columnMapping = (condition) => { mapping = condition; return action; };
|
|
540
|
+
action.queued = (condition = true) => { queued = condition; return action; };
|
|
541
|
+
return action;
|
|
542
|
+
}
|
|
543
|
+
}
|