kempo-inventory 0.2.0

package/server/utils/importExport.js

@@ -0,0 +1,239 @@
+import db from 'kempo/server/db/index.js';
+import { asc } from 'drizzle-orm';
+import { kempoInventoryItem, kempoInventoryCategory } from '../db/schema.js';
+import { getFields } from './fields.js';
+import { getItemBySku, createItem, updateItem, adjustStock } from './items.js';
+import { getCategories, saveCategory } from './categories.js';
+import { applicableFields, coerceValues } from './fieldTypes.js';
+import { normalizeTags } from './tags.js';
+import { tidyCategory, categoryKey, MAX_CATEGORY } from './categoryLogic.js';
+import { readMedia } from './media.js';
+
+/*
+  Exporting and importing an inventory.
+
+  The export is one JSON document (see FORMAT below), on its own for "data only" or inside a zip together
+  with every photo and file it uses ("with media"). Import reads the same document.
+
+  What it carries: every item (SKU, name, category, quantity, tags, description and the values of
+  its custom fields) and every category (description, tags, picture). It also lists the field
+  definitions, for reference: an import never creates or changes fields, so fields have to exist
+  already for their values to come across. Values for fields that do not exist here, or do not apply to an
+  item's category, are ignored and counted.
+
+  Photo and file fields only travel in the zip version. A field linking to another item is written as
+  that item's SKU and resolved by SKU on import.
+*/
+export const FORMAT = 'kempo-inventory-export';
+export const VERSION = 1;
+export const MAX_IMPORT_ROWS = 500;
+
+const safeName = name => String(name ?? 'file').replace(/[\\/:*?"<>|\u0000-\u001f]+/g, '_').replace(/^\.+/, '').slice(0, 80) || 'file';
+
+/*
+  Returns { data, files }: the JSON document and, with \`withMedia\`, the media files it refers to as
+  [{ path, bytes }]. Media references in the document are paths within the zip ("media/3-photo.jpg"),
+  described in \`data.media\`.
+*/
+export const buildExport = async ({ withMedia = false } = {}) => {
+  const [fieldsError, fields] = await getFields();
+  if(fieldsError) return [fieldsError, null];
+
+  const itemRows = await db.select().from(kempoInventoryItem).orderBy(asc(kempoInventoryItem.sku));
+  const categoryRows = await db.select().from(kempoInventoryCategory).orderBy(asc(kempoInventoryCategory.name));
+  const skuById = new Map(itemRows.map(row => [row.id, row.sku]));
+
+  /* The media each item holds, in the fields that apply to it. */
+  const mediaIdsOf = row => applicableFields(fields, row.category)
+    .filter(f => f.type === 'media')
+    .flatMap(f => (row.data?.[f.key] ?? []));
+  const wanted = [];
+  if(withMedia){
+    for(const row of itemRows) for(const id of mediaIdsOf(row)) if(!wanted.includes(id)) wanted.push(id);
+    for(const row of categoryRows) if(row.image && !wanted.includes(row.image)) wanted.push(row.image);
+  }
+
+  const pathOf = new Map();
+  const files = [];
+  const media = {};
+  let missingMedia = 0;
+  for(const id of wanted){
+    const [error, file] = await readMedia(id);
+    if(error){ missingMedia++; continue; }
+    const path = `media/${pathOf.size + 1}-${safeName(file.name)}`;
+    pathOf.set(id, path);
+    files.push({ path, bytes: file.bytes });
+    media[path] = { name: file.name, kind: file.kind, alt: file.alt };
+  }
+
+  const items = itemRows.map(row => {
+    const values = {};
+    for(const field of applicableFields(fields, row.category)){
+      const value = row.data?.[field.key];
+      if(value === undefined || value === null) continue;
+      if(field.type === 'media'){
+        const paths = (value ?? []).map(id => pathOf.get(id)).filter(Boolean);
+        if(paths.length) values[field.key] = paths;
+      } else if(field.type === 'item'){
+        const sku = skuById.get(value);
+        if(sku) values[field.key] = sku;
+      } else {
+        values[field.key] = value;
+      }
+    }
+    return { sku: row.sku, name: row.name, category: row.category, quantity: row.quantity, tags: row.tags ?? [], description: row.description, fields: values };
+  });
+
+  const categories = categoryRows.map(row => ({
+    name: row.name,
+    description: row.description,
+    tags: row.tags ?? [],
+    ...(row.image && pathOf.has(row.image) ? { image: pathOf.get(row.image) } : {}),
+  }));
+
+  const data = {
+    format: FORMAT,
+    version: VERSION,
+    exportedAt: new Date().toISOString(),
+    withMedia,
+    fields: fields.map(f => ({ key: f.key, label: f.label, type: f.type, category: f.categoryName ?? '', required: f.required, options: f.options })),
+    categories,
+    items,
+    ...(withMedia ? { media } : {}),
+  };
+  return [null, { data, files, missingMedia }];
+};
+
+const given = value => value !== undefined && value !== null && value !== '' && !(Array.isArray(value) && !value.length);
+
+/*
+  Applies (or, with \`dryRun\`, only checks) a batch of items and categories from an export.
+
+    onMatch  'skip'    an item or category that already exists (same SKU / same name) is left alone
+             'update'  it is updated with the values the file provides; a value the file leaves empty
+                       is never cleared. A changed quantity is applied as a stock adjustment, so the
+                       history shows it.
+
+  Rows are independent: one that fails does not stop the others. Resolves to
+  [null, { results, categoryResults, summary }].
+*/
+export const importData = async ({ items = [], categories = [], onMatch = 'skip', dryRun = false } = {}, { userId = '', canUpdate = false, canAdjustStock = false, maxRows = MAX_IMPORT_ROWS } = {}) => {
+  if(!['skip', 'update'].includes(onMatch)) return [{ code: 400, msg: 'onMatch must be "skip" or "update"' }, null];
+  if(!Array.isArray(items) || !Array.isArray(categories)) return [{ code: 400, msg: 'items and categories must be lists' }, null];
+  if(items.length > maxRows || categories.length > maxRows) return [{ code: 413, msg: `Send at most ${maxRows} rows at a time` }, null];
+  if(onMatch === 'update' && !canUpdate) return [{ code: 403, msg: 'Updating existing items needs the items:update permission' }, null];
+
+  const [fieldsError, fields] = await getFields();
+  if(fieldsError) return [fieldsError, null];
+
+  const summary = { created: 0, updated: 0, skipped: 0, errors: 0, ignoredValues: 0, categoriesCreated: 0, categoriesUpdated: 0, categoriesSkipped: 0 };
+  const results = [];
+  const deferredLinks = [];   // { result, sku, key, linkSku }: links to an item that does not exist yet
+
+  const fail = (result, message) => { result.action = 'error'; result.message = message; summary.errors++; return result; };
+
+  /* Categories first: whether one already exists is decided before this import's own items give it implicit existence. */
+  const categoryResults = [];
+  const [, existingCategories] = await getCategories({ includeEmpty: true });
+  const known = new Set((existingCategories?.categories ?? []).map(c => c.key));
+  for(const [index, row] of categories.entries()){
+    const name = tidyCategory(row?.name);
+    const result = { row: index + 1, name, action: '', message: '' };
+    categoryResults.push(result);
+    if(!name){ result.action = 'error'; result.message = 'Missing name'; summary.errors++; continue; }
+    const exists = known.has(categoryKey(name));
+    if(exists && onMatch === 'skip'){ result.action = 'skip'; summary.categoriesSkipped++; continue; }
+    result.action = exists ? 'update' : 'create';
+    if(dryRun){ exists ? summary.categoriesUpdated++ : summary.categoriesCreated++; continue; }
+    const changes = {};
+    if(given(row.description)) changes.description = String(row.description);
+    if(given(row.tags)) changes.tags = row.tags;
+    if(given(row.image)) changes.image = row.image;
+    const [error] = await saveCategory(name, changes, { userId });
+    if(error){ result.action = 'error'; result.message = error.msg; summary.errors++; continue; }
+    known.add(categoryKey(name));
+    exists ? summary.categoriesUpdated++ : summary.categoriesCreated++;
+  }
+
+  for(const [index, row] of items.entries()){
+    const result = { row: index + 1, sku: String(row?.sku ?? '').trim(), name: String(row?.name ?? '').trim(), action: '', message: '' };
+    results.push(result);
+    if(!row || typeof row !== 'object') { fail(result, 'Not an item'); continue; }
+    if(!result.sku){ fail(result, 'Missing SKU'); continue; }
+
+    const [, existing] = await getItemBySku(result.sku);
+    if(existing && onMatch === 'skip'){ result.action = 'skip'; result.message = 'Already exists'; summary.skipped++; continue; }
+
+    const category = given(row.category) ? tidyCategory(row.category) : (existing?.category ?? '');
+    if(category.length > MAX_CATEGORY){ fail(result, `Category must be ${MAX_CATEGORY} characters or fewer`); continue; }
+    let tags;
+    if(given(row.tags)){
+      const [tagsError, cleanTags] = normalizeTags(row.tags);
+      if(tagsError){ fail(result, tagsError.msg); continue; }
+      tags = cleanTags;
+    }
+    let quantity;
+    if(given(row.quantity)){
+      quantity = Number(row.quantity);
+      if(!Number.isInteger(quantity) || quantity < 0){ fail(result, 'Quantity must be a non-negative whole number'); continue; }
+    }
+    if(!existing && !result.name){ fail(result, 'Missing name'); continue; }
+
+    /* Custom field values: only fields that exist and apply to the item's category. */
+    const applicable = applicableFields(fields, category);
+    const byKey = new Map(applicable.map(f => [f.key, f]));
+    const values = {};
+    const links = [];
+    for(const [key, value] of Object.entries(row.fields && typeof row.fields === 'object' ? row.fields : {})){
+      const field = byKey.get(key);
+      if(!field){ summary.ignoredValues++; continue; }
+      if(!given(value)) continue;
+      if(field.type === 'item'){
+        const [, target] = await getItemBySku(String(value).trim());
+        if(target) values[key] = target.id; else links.push({ key, linkSku: String(value).trim() });
+      } else {
+        values[key] = value;
+      }
+    }
+    const [invalid, coerced] = coerceValues(applicable, values, { partial: Boolean(existing) });
+    if(invalid){ fail(result, invalid.msg); continue; }
+
+    if(existing){
+      if(quantity !== undefined && quantity !== existing.quantity && !canAdjustStock){ fail(result, 'Changing a quantity needs the stock:adjust permission'); continue; }
+      result.action = 'update';
+      if(dryRun){ summary.updated++; continue; }
+      const changes = { fields: coerced };
+      if(given(row.name)) changes.name = result.name;
+      if(given(row.description)) changes.description = String(row.description);
+      if(given(row.category)) changes.category = category;
+      if(tags !== undefined) changes.tags = tags;
+      const [error] = await updateItem(existing.id, { ...changes, userId });
+      if(error && !(error.code === 400 && error.msg === 'No changes provided')){ fail(result, error.msg); continue; }
+      if(quantity !== undefined && quantity !== existing.quantity){
+        const [adjustError] = await adjustStock(existing.id, { delta: quantity - existing.quantity, reason: 'import', note: 'Imported', userId });
+        if(adjustError){ fail(result, adjustError.msg); continue; }
+      }
+      summary.updated++;
+      for(const link of links) deferredLinks.push({ result, itemId: existing.id, ...link });
+    } else {
+      result.action = 'create';
+      if(dryRun){ summary.created++; continue; }
+      const [error, created] = await createItem({
+        sku: result.sku, name: result.name, description: given(row.description) ? String(row.description) : '',
+        category, tags: tags ?? [], quantity: quantity ?? 0, fields: coerced, userId,
+      });
+      if(error){ fail(result, error.msg); continue; }
+      summary.created++;
+      for(const link of links) deferredLinks.push({ result, itemId: created.id, ...link });
+    }
+  }
+
+  /* Links to items created later in the same import. */
+  for(const link of deferredLinks){
+    const [, target] = await getItemBySku(link.linkSku);
+    if(!target){ link.result.message = `${link.result.message ? link.result.message + '; ' : ''}linked item ${link.linkSku} was not found`; continue; }
+    await updateItem(link.itemId, { fields: { [link.key]: target.id }, userId });
+  }
+
+  return [null, { results, categoryResults, summary }];
+};

package/server/utils/importFile.js

@@ -0,0 +1,148 @@
+import { readZip } from './zipRead.js';
+import { importData, FORMAT } from './importExport.js';
+import { storeMedia } from './media.js';
+import { refuseMedia, safeFileName } from './mediaSafety.js';
+
+/*
+  Importing an uploaded export (a .json or a .zip), entirely on the server.
+
+  Nothing here may be reached by anyone but the top tier: the routes check `fields:manage` before they
+  read a byte of the upload, and these functions assume it. What they do with the upload:
+
+    - A zip is recognised by its first bytes, not its name, and read in memory by zipRead.js: entries are
+      looked up by the exact names the inventory.json asks for, never written to disk by name, with a
+      cap on how many files it may hold and how far one may inflate.
+    - A photo reference must have the form "media/<name>" (no folders, no "..", no slashes). Anything else
+      is left as it is and the row fails validation, so no name from the zip is ever used as a path.
+    - A file is only stored if its extension is a photo, video, audio or PDF type AND its first bytes really
+      are that type (mediaSafety.js). Everything else in the zip is skipped and reported.
+    - Files are stored through the normal library routines (kempo-files or kempo-media), as the person doing
+      the import, so those libraries apply their own rules (untrusted files are never executed).
+*/
+export const MAX_ITEMS = 50000;
+export const MAX_MEDIA_FILES = 5000;
+export const MAX_MEDIA_BYTES = 1024 * 1024 * 1024;   // all the photos in one import
+const MAX_JSON_BYTES = 100 * 1024 * 1024;
+const MEDIA_PATH = /^media\/[^/\\\u0000]{1,200}$/;
+
+const isZip = bytes => bytes.length >= 4 && bytes[0] === 0x50 && bytes[1] === 0x4B && ((bytes[2] === 3 && bytes[3] === 4) || (bytes[2] === 5 && bytes[3] === 6));
+
+const isMediaPaths = value => Array.isArray(value) && value.length > 0 && value.every(v => typeof v === 'string' && MEDIA_PATH.test(v));
+
+/* Opens the upload: { data, zip } or an { code, msg } fit to show a person. */
+export const parseImportFile = async bytes => {
+  let zip = null;
+  let text;
+  try {
+    if(isZip(bytes)){
+      zip = await readZip(bytes);
+      text = await zip.text('inventory.json');
+      if(text === null) return [{ code: 400, msg: 'This zip has no inventory.json. Is it an export from this inventory?' }, null];
+    } else {
+      if(bytes.length > MAX_JSON_BYTES) return [{ code: 413, msg: 'This file is too large to import' }, null];
+      text = bytes.toString('utf8');
+    }
+  } catch(error) {
+    return [{ code: 400, msg: error.message || 'This zip file could not be read' }, null];
+  }
+  if(text.length > MAX_JSON_BYTES) return [{ code: 413, msg: 'inventory.json is too large to import' }, null];
+  let data;
+  try {
+    data = JSON.parse(text);
+  } catch {
+    return [{ code: 400, msg: 'This file is not valid JSON.' }, null];
+  }
+  if(Array.isArray(data)) data = { items: data };
+  if(!data || typeof data !== 'object' || !Array.isArray(data.items)) return [{ code: 400, msg: 'This file does not look like an inventory export (it has no list of items).' }, null];
+  if(data.format && data.format !== FORMAT) return [{ code: 400, msg: 'This file is not an inventory export.' }, null];
+  if(data.items.length > MAX_ITEMS) return [{ code: 413, msg: `This file has more than ${MAX_ITEMS} items` }, null];
+  if(data.categories !== undefined && !Array.isArray(data.categories)) return [{ code: 400, msg: 'categories must be a list' }, null];
+  return [null, { data: { ...data, categories: data.categories ?? [] }, zip }];
+};
+
+/* The rows without their photo references, which cannot be checked until the photos are stored. */
+const withoutMedia = ({ items, categories }) => ({
+  items: items.map(item => ({ ...item, fields: Object.fromEntries(Object.entries(item?.fields && typeof item.fields === 'object' ? item.fields : {}).filter(([, value]) => !isMediaPaths(value))) })),
+  categories: categories.map(row => { const { image, ...rest } = row ?? {}; return rest; }),
+});
+
+const mediaIn = ({ items, categories }) => {
+  const paths = new Set();
+  for(const item of items) for(const value of Object.values(item?.fields ?? {})) if(isMediaPaths(value)) value.forEach(p => paths.add(p));
+  for(const category of categories) if(typeof category?.image === 'string' && MEDIA_PATH.test(category.image)) paths.add(category.image);
+  return paths;
+};
+
+const PERMISSIONS = { canUpdate: true, canAdjustStock: true, maxRows: MAX_ITEMS };   // the caller is the top tier, which holds all of these
+
+/* What importing this file would do, changing nothing: { results, categoryResults, summary, media }. */
+export const previewImport = async (bytes, { userId }) => {
+  const [parseError, parsed] = await parseImportFile(bytes);
+  if(parseError) return [parseError, null];
+  const [error, report] = await importData({ ...withoutMedia(parsed.data), onMatch: 'update', dryRun: true }, { userId, ...PERMISSIONS });
+  if(error) return [error, null];
+  const paths = mediaIn(parsed.data);
+  return [null, {
+    ...report,
+    counts: { items: parsed.data.items.length, categories: parsed.data.categories.length },
+    withMedia: Boolean(parsed.zip),
+    mediaFiles: paths.size,
+  }];
+};
+
+/* Does the import: stores the photos the rows to be written need, then writes the rows. */
+export const applyImport = async (bytes, { onMatch, userId }) => {
+  const [parseError, parsed] = await parseImportFile(bytes);
+  if(parseError) return [parseError, null];
+  const { data, zip } = parsed;
+  const items = structuredClone(data.items);
+  const categories = structuredClone(data.categories);
+
+  const stored = new Map();       // path -> stored id, or null when the file was refused
+  const skippedFiles = [];
+  let uploaded = 0;
+  let uploadedBytes = 0;
+  const idFor = async path => {
+    if(stored.has(path)) return stored.get(path);
+    stored.set(path, null);
+    const name = safeFileName(data.media?.[path]?.name || path.split('/').pop());
+    const file = zip ? await zip.read(path).catch(error => error) : null;
+    if(file instanceof Error){ skippedFiles.push({ name, reason: file.message }); return null; }
+    if(!file){ skippedFiles.push({ name, reason: 'not in the zip' }); return null; }
+    const reason = refuseMedia(name, file);
+    if(reason){ skippedFiles.push({ name, reason }); return null; }
+    if(uploaded >= MAX_MEDIA_FILES || uploadedBytes + file.length > MAX_MEDIA_BYTES){ skippedFiles.push({ name, reason: 'over the size limit for one import' }); return null; }
+    const [error, id] = await storeMedia({ name, bytes: file, altText: data.media?.[path]?.alt ?? '', userId });
+    if(error){ skippedFiles.push({ name, reason: error.msg }); return null; }
+    uploaded++;
+    uploadedBytes += file.length;
+    stored.set(path, id);
+    return id;
+  };
+
+  /* Photos only for the rows that will actually be created or updated. */
+  const [planError, plan] = await importData({ ...withoutMedia({ items, categories }), onMatch, dryRun: true }, { userId, ...PERMISSIONS });
+  if(planError) return [planError, null];
+  const wanted = new Set(plan.results.filter(r => r.action === 'create' || r.action === 'update').map(r => r.row));
+  const wantedCategories = new Set(plan.categoryResults.filter(r => r.action === 'create' || r.action === 'update').map(r => r.row));
+
+  for(const [index, item] of items.entries()){
+    if(!item || typeof item.fields !== 'object') continue;
+    for(const [key, value] of Object.entries(item.fields)){
+      if(!isMediaPaths(value)) continue;
+      if(!wanted.has(index + 1)){ delete item.fields[key]; continue; }
+      const ids = [];
+      for(const path of value){ const id = await idFor(path); if(id) ids.push(id); }
+      if(ids.length) item.fields[key] = ids; else delete item.fields[key];
+    }
+  }
+  for(const [index, category] of categories.entries()){
+    if(typeof category?.image !== 'string') continue;
+    const id = MEDIA_PATH.test(category.image) && wantedCategories.has(index + 1) ? await idFor(category.image) : null;
+    if(id) category.image = id; else delete category.image;
+  }
+
+  const [error, report] = await importData({ items, categories, onMatch, dryRun: false }, { userId, ...PERMISSIONS });
+  if(error) return [error, null];
+  return [null, { ...report, uploaded, skippedFiles }];
+};

package/server/utils/importStage.js

@@ -0,0 +1,76 @@
+import { mkdir, writeFile, readFile, readdir, rm, stat } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import crypto from 'crypto';
+
+/*
+  Holds an uploaded import file between the preview and the import, so the person uploads it once.
+
+  Only a top-tier user ever gets here (the routes check first). The file is written under the system's
+  temp directory with a random 128-bit id as its whole name, readable only by the server's own user. It is
+  never served, never opened by its content's names, and only the person who staged it can use it. It is
+  deleted when the import runs, when they stage another, or after 30 minutes.
+*/
+const DIR = () => join(tmpdir(), 'kempo-inventory-import');
+const TTL_MS = 30 * 60 * 1000;
+const MAX_PER_USER = 2;
+const ID = /^[a-f0-9]{32}$/;
+
+const metaPath = id => join(DIR(), `${id}.json`);
+const dataPath = id => join(DIR(), `${id}.bin`);
+
+const remove = async id => {
+  await rm(dataPath(id), { force: true }).catch(() => {});
+  await rm(metaPath(id), { force: true }).catch(() => {});
+};
+
+const readMeta = async id => {
+  try {
+    return JSON.parse(await readFile(metaPath(id), 'utf8'));
+  } catch {
+    return null;
+  }
+};
+
+/* Deletes everything expired, and a user's oldest files beyond the limit. */
+export const sweep = async (userId = null, now = Date.now()) => {
+  let names;
+  try { names = await readdir(DIR()); } catch { return; }
+  const mine = [];
+  for(const name of names){
+    const match = /^([a-f0-9]{32})\.json$/.exec(name);
+    if(!match) continue;
+    const meta = await readMeta(match[1]);
+    if(!meta || now - meta.createdAt > TTL_MS) await remove(match[1]);
+    else if(userId && meta.userId === userId) mine.push({ id: match[1], createdAt: meta.createdAt });
+  }
+  mine.sort((a, b) => b.createdAt - a.createdAt);
+  for(const old of mine.slice(MAX_PER_USER - 1)) await remove(old.id);
+};
+
+/* Stores the bytes for `userId` and returns the id to use for the import. */
+export const stage = async (bytes, userId, name = '') => {
+  await mkdir(DIR(), { recursive: true, mode: 0o700 });
+  await sweep(userId);
+  const id = crypto.randomBytes(16).toString('hex');
+  await writeFile(dataPath(id), bytes, { mode: 0o600 });
+  await writeFile(metaPath(id), JSON.stringify({ userId, name: String(name).slice(0, 200), createdAt: Date.now() }), { mode: 0o600 });
+  return id;
+};
+
+/* The staged bytes, or null when the id is malformed, unknown, expired, or belongs to someone else. */
+export const readStaged = async (id, userId) => {
+  if(typeof id !== 'string' || !ID.test(id)) return null;
+  const meta = await readMeta(id);
+  if(!meta || meta.userId !== userId || Date.now() - meta.createdAt > TTL_MS) return null;
+  try {
+    await stat(dataPath(id));
+    return { bytes: await readFile(dataPath(id)), name: meta.name };
+  } catch {
+    return null;
+  }
+};
+
+export const discardStaged = async id => {
+  if(typeof id === 'string' && ID.test(id)) await remove(id);
+};