@jwilleke/ngdpbase-types 5.25.0 → 5.27.0

package/lib/src/managers/ImportManager.d.ts

@@ -0,0 +1,590 @@
+/**
+ * Import Manager
+ *
+ * Manages the import of content from external wiki formats into ngdpbase.
+ * Uses an extensible converter registry pattern to support multiple formats
+ * (JSPWiki, MediaWiki, Confluence, etc.).
+ *
+ * @module ImportManager
+ *
+ * @example
+ * const importManager = engine.getManager('ImportManager');
+ *
+ * // Register additional converters
+ * importManager.registerConverter(new MediaWikiConverter());
+ *
+ * // Preview import
+ * const preview = await importManager.previewImport({
+ *   sourceDir: '/path/to/wiki',
+ *   format: 'auto'
+ * });
+ *
+ * // Execute import
+ * const result = await importManager.importPages({
+ *   sourceDir: '/path/to/wiki',
+ *   format: 'jspwiki',
+ *   dryRun: false
+ * });
+ */
+import { type ActorContext } from '../context/ActorContext.js';
+import BaseManager, { BackupData } from './BaseManager.js';
+import type { WikiEngine } from '../types/WikiEngine.js';
+import { IContentConverter } from '../converters/IContentConverter.js';
+import { type ZipReadLimits } from '../utils/zipArchive.js';
+import { type JobContext } from '../context/JobContext.js';
+import type { IceLedger, IceReceiver, IceRunReport, IceRunRequest, IceShim, RegisteredShim } from '../ice/types.js';
+/**
+ * Options for import operations
+ */
+export interface ImportOptions {
+    /** Source directory containing files to import */
+    sourceDir: string;
+    /**
+     * Target directory for converted files (default: the live pages dir).
+     * When this resolves to the live pages directory, imports go through
+     * PageManager save semantics (#880 — indexed, versioned, in Recent
+     * Changes). Any other directory means convert-to-files/export mode:
+     * raw markdown files are written there instead.
+     */
+    targetDir?: string;
+    /** Format ID or 'auto' for auto-detection */
+    format?: string;
+    /** Keep original files after conversion (default: true) */
+    preserveOriginals?: boolean;
+    /** Preview only, don't write files (default: false) */
+    dryRun?: boolean;
+    /** Generate UUIDs for pages that don't have them (default: true) */
+    generateUUIDs?: boolean;
+    /** File extensions to process (default: determined by format) */
+    fileExtensions?: string[];
+    /** Maximum files to process (for large imports) */
+    limit?: number;
+    /** Skip first N files (for resuming imports) */
+    offset?: number;
+    /** Progress callback for streaming updates */
+    onProgress?: (event: ImportProgressEvent) => void;
+    /**
+     * Categorises this run for the per-run summary (#738). Examples:
+     * `'paste'`, `'url'`, `'file'`, `'jspwiki'`, `'ingest:<sourceId>'`.
+     * Defaults to `'import'` when caller doesn't specify.
+     */
+    importType?: string;
+    /**
+     * The identity that initiated this run, forwarded rather than flattened
+     * (#1164, #1179, #1236).
+     *
+     * Until #1236 this sat beside `actor?: string` and `actorIsSystem?`, two
+     * more carriers of the same fact that the run summary and the page
+     * frontmatter read instead. One identity now: the run's actor, the page's
+     * `editor` and its `author` fallback, and the summary's `isSystem` are all
+     * derived from this context. A username string cannot carry a
+     * delegation. `importPageAttachments` needed a context to authorise an
+     * upload, could not get one from a string, and __fabricated__ one:
+     *
+     *     context: { username: author, name: author, isAuthenticated: true,
+     *                roles: ['admin'] }
+     *
+     * where `author` came from the imported file's own `attachment.properties`.
+     * The PDP (then `UserManager.hasPermission`) trusts supplied roles verbatim — it does not
+     * re-resolve them — so that object authorised as an admin, attributed the
+     * write to a name taken from imported content, and carried no `viaToken`,
+     * so an admin's agent token doing an import escaped its scope ceiling.
+     *
+     * That is P1 twice over: a parameter that could not carry provenance, and a
+     * synthetic admin principal invented to fill the gap it left.
+     *
+     * Mandatory: the upload door records it, the summary names it, the page
+     * carries it. A non-request caller passes a JobContext with its reason.
+     */
+    actorContext: ActorContext;
+    /**
+     * What to do when a file's title matches an existing page (#874).
+     * `'skip'` (default) — leave the existing page untouched, report duplicate.
+     * `'overwrite'` — update the existing page in place through PageManager save
+     * semantics: UUID, author, created, and slug preserved; version bumped by
+     * the provider; `editor` = `actor`; search index and link graph updated
+     * in-band (same contract as an edit-form save / the ingest API upsert).
+     * Folder import only — URL import keeps hard-skip.
+     */
+    conflictPolicy?: 'skip' | 'overwrite';
+    /**
+     * Explicit private-store destination (#1389). Markdown still goes through
+     * PageManager. JSON/XML become files in the store via AttachmentManager.
+     * Never inferred from a page. Store id from ConfigurationManager when omitted.
+     */
+    private?: boolean;
+    store?: string;
+}
+/**
+ * Progress event for streaming import updates
+ */
+export interface ImportProgressEvent {
+    /** Event type */
+    type: 'start' | 'progress' | 'complete' | 'error';
+    /** Source file path */
+    file?: string;
+    /** Current file index (0-based) */
+    index?: number;
+    /** Total files to process */
+    total?: number;
+    /** Status of the file import */
+    status?: 'success' | 'skipped' | 'failed';
+    /** Error message if failed */
+    error?: string;
+    /** Imported page title (success only) — lets the UI link to /view/<title> */
+    pageTitle?: string;
+    /** Final result (for complete event) */
+    result?: unknown;
+}
+/**
+ * Error information for failed imports
+ */
+export interface ImportError {
+    /** Source file path */
+    file: string;
+    /** Error message */
+    message: string;
+    /** Error stack trace (optional) */
+    stack?: string;
+}
+/**
+ * Information about an imported file
+ */
+export interface ImportedFile {
+    /** Original source file path */
+    sourcePath: string;
+    /** Target file path */
+    targetPath: string;
+    /** Format detected/used */
+    format: string;
+    /** File size in bytes */
+    size: number;
+    /** Extracted metadata */
+    metadata: Record<string, unknown>;
+    /** Conversion warnings */
+    warnings: string[];
+    /** Whether file was actually written (false for dry runs or duplicates) */
+    written: boolean;
+    /** Reason the file was skipped (e.g. 'duplicate') */
+    skippedReason?: string;
+    /** UUID of existing page if duplicate */
+    existingPageUuid?: string;
+    /** True when an existing page was (or on dry run: would be) updated in place (#874) */
+    overwritten?: boolean;
+    /** Attachment import stats (JSPWiki imports only) */
+    attachments?: {
+        imported: number;
+        skipped: number;
+        errors: string[];
+    };
+}
+/**
+ * Result of an import operation
+ */
+export interface ImportResult {
+    /** Overall success status */
+    success: boolean;
+    /** Number of files converted */
+    converted: number;
+    /** Number of files skipped */
+    skipped: number;
+    /** Number of files with errors */
+    failed: number;
+    /** Total files processed */
+    total: number;
+    /** Error details */
+    errors: ImportError[];
+    /** Information about imported files */
+    files: ImportedFile[];
+    /** Import duration in milliseconds */
+    durationMs: number;
+}
+/**
+ * Why a takeout import was refused before anything was written (#1472).
+ * A refusal is never a partial import.
+ */
+export declare class TakeoutImportRefused extends Error {
+    readonly reason: 'not-owner' | 'no-such-store' | 'locked' | 'unreadable';
+    constructor(reason: 'not-owner' | 'no-such-store' | 'locked' | 'unreadable', message: string);
+}
+/** One page's outcome in a takeout import (#1472). */
+export type TakeoutPageOutcome = {
+    title: string;
+    outcome: 'imported';
+    importedAs: string;
+} | {
+    title: string;
+    outcome: 'unchanged';
+} | {
+    title: string;
+    outcome: 'changed-since-takeout';
+}
+/** `where` is named only when the requester may view that page. */
+ | {
+    title: string;
+    outcome: 'uuid-elsewhere';
+    where?: string;
+} | {
+    title: string;
+    outcome: 'failed';
+    message: string;
+};
+export type TakeoutImportReport = {
+    store: string;
+    pages: TakeoutPageOutcome[];
+    /** Files stored, or matched to one the store already held. */
+    files: number;
+    /** Files that could not be stored, by name. */
+    fileErrors: Array<{
+        name: string;
+        message: string;
+    }>;
+    /**
+     * Files stored that no page now in the store links to — typically because
+     * the pages that used them were skipped (operator, 2026-09-25: they come
+     * in, and are reported). Named here, for the owner; counted in the logs.
+     */
+    unlinkedFiles: string[];
+    /** Archive members that are neither a page nor a file. */
+    ignored: string[];
+    /**
+     * The private page this import's report was saved as, in the target store
+     * (operator, 2026-09-25). Absent only when that save failed.
+     */
+    reportPage?: string;
+};
+/**
+ * Import Manager class
+ *
+ * Manages content import with extensible format support via converter registry.
+ */
+declare class ImportManager extends BaseManager {
+    /** Registry of format converters */
+    private converterRegistry;
+    /** ICE (#1979): shims by id (`<owner>.<name>`), in registration order. */
+    private readonly shimRegistry;
+    /** ICE: receivers by id. */
+    private readonly receiverRegistry;
+    /** ICE: runs this process knows about, oldest first. */
+    private readonly iceRuns;
+    /** ICE: runs waiting for the background job, each with its own context. */
+    private readonly iceQueue;
+    /** ICE: true while a background job is draining the queue. */
+    private iceDraining;
+    /** ICE: the last background run id the queue was handed to. */
+    private iceLastJobRunId;
+    private iceLedger;
+    constructor(engine: WikiEngine);
+    /**
+     * Initialize the manager
+     */
+    initialize(config?: Record<string, unknown>): Promise<void>;
+    /**
+     * Register a content converter. Since #1979 (D8) it is also the ICE shim
+     * `core.<formatId>`, producing NCM for the pages receiver, so the folder
+     * import and the engine run the same conversion.
+     *
+     * @param converter - Converter instance implementing IContentConverter
+     */
+    registerConverter(converter: IContentConverter): void;
+    /**
+     * Register a shim (D6, D7). Core registers its own as owner `core`; an
+     * add-on registers through `host.registerShim`, which names the add-on as
+     * the owner and refuses a shim its manifest does not declare.
+     *
+     * @returns The shim's id, `<owner>.<name>`
+     * @throws Error when the shim is malformed or its id is taken
+     */
+    registerShim(owner: string, shim: IceShim): string;
+    /**
+     * Register a receiver (D4): a door that already exists. Core's are
+     * `core.pages` and `core.files`; an add-on registers its own manager's door
+     * through `host.registerReceiver`.
+     *
+     * @returns The receiver's id, `<owner>.<name>`
+     */
+    registerReceiver(owner: string, receiver: IceReceiver): string;
+    /** Is this shim switched on (D6)? `ngdpbase.import.shims.<id>.enabled`, default on. */
+    isShimEnabled(id: string): boolean;
+    /** Every registered shim, and whether it is on. */
+    listShims(): Array<Pick<RegisteredShim, 'id' | 'owner' | 'label' | 'direction' | 'produces' | 'receivers'> & {
+        enabled: boolean;
+    }>;
+    /** The receivers this context may use (D5): each answers with its own existing permission. */
+    receiversFor(ctx: ActorContext): Promise<Array<{
+        id: string;
+        label?: string;
+    }>>;
+    /**
+     * Run one import now and wait for it. For code already off the request
+     * path: an add-on's own background job, a test. A route uses
+     * {@link enqueueImport}, so the work runs off the request path.
+     *
+     * @param ctx - Who it runs as: the person's context, or a job's (P1)
+     */
+    runImport(request: IceRunRequest, ctx: ActorContext): Promise<IceRunReport>;
+    /**
+     * Queue an import as background work and return its run id at once. The
+     * run carries `ctx` — the person who asked (`jobContextFromRequest`), or a
+     * job's context with its reason — and each receiver checks its own
+     * permission with it when the run reaches it (D5).
+     */
+    enqueueImport(request: IceRunRequest, ctx: JobContext): Promise<string>;
+    /** A run's report, to the person it runs as; null to anyone else. */
+    getImportRun(runId: string, ctx: ActorContext): IceRunReport | null;
+    /** Use this ledger instead of the file one: a test's, or a database table's. */
+    setImportLedger(ledger: IceLedger): void;
+    private newRunReport;
+    private executeRun;
+    private enabledShims;
+    /**
+     * Hand the queue to the background job. One job run drains every queued
+     * import, each under its own context. A job run that has finished draining
+     * but is not yet marked done answers `enqueue` with its old run id; then
+     * try again shortly, so nothing queued is stranded.
+     */
+    private kickIceQueue;
+    private setting;
+    private numberSetting;
+    /** `ngdpbase.import.engine.*`: every limit the engine applies is configuration. */
+    private engineSettings;
+    /** The engine's ledger (fingerprints, per-file status, item keys), opened on first use. */
+    private ledger;
+    /** What a shim is handed: the engine's guarded fetch (design rule) and its own options (D6). */
+    private shimTools;
+    /** A receiver's existing permission, asked of the PDP with the run's context (D5, P2). */
+    private permits;
+    /**
+     * `core.pages` (D4): NCM into pages, through PageManager's save door, as the
+     * folder import saves. It checks `page-create` itself (D5). A title that
+     * already exists is a duplicate, as it is for the folder import by default.
+     */
+    private pagesReceiver;
+    /**
+     * `core.files` (D4): a file through AttachmentManager's upload door, which
+     * checks `asset-upload` itself. Also where a kept original goes (D11) when
+     * the chosen receiver does not keep its own. It becomes FileManager's write
+     * door once the files layer has one (#1851).
+     */
+    private filesReceiver;
+    /**
+     * Get available format IDs
+     *
+     * @returns Array of registered format identifiers
+     */
+    getAvailableFormats(): string[];
+    /**
+     * Get converter by format ID
+     *
+     * @param formatId - Format identifier
+     * @returns Converter instance or undefined
+     */
+    getConverter(formatId: string): IContentConverter | undefined;
+    /**
+     * Get all registered converters with their metadata
+     *
+     * @returns Array of converter info objects
+     */
+    getConverterInfo(): Array<{
+        formatId: string;
+        formatName: string;
+        fileExtensions: string[];
+    }>;
+    /**
+     * Auto-detect format from file content
+     *
+     * @param content - File content
+     * @param filename - Filename (for extension matching)
+     * @returns Format ID or null if no match
+     */
+    detectFormat(content: string, filename: string): string | null;
+    /**
+     * Preview import without writing files
+     *
+     * @param options - Import options (dryRun is forced to true)
+     * @returns Import result with preview data
+     */
+    previewImport(options: ImportOptions): Promise<ImportResult>;
+    /**
+     * Import pages with streaming progress updates
+     * Alias for importPages that makes the progress callback more explicit
+     *
+     * @param options - Import options with onProgress callback
+     * @returns Import result with statistics and file details
+     */
+    importPagesWithProgress(options: ImportOptions): Promise<ImportResult>;
+    /**
+     * Import pages from source directory
+     *
+     * @param options - Import options
+     * @returns Import result with statistics and file details
+     */
+    importPages(options: ImportOptions): Promise<ImportResult>;
+    /**
+     * Import a single page
+     *
+     * @param filePath - Source file path
+     * @param options - Import options
+     * @returns Imported file info or null if skipped
+     */
+    importSinglePage(filePath: string, options: ImportOptions): Promise<ImportedFile | null>;
+    /**
+     * Update an existing page in place from an import (#874 conflict policy
+     * 'overwrite'). Mirrors the edit-form / ingest-API save contract: UUID,
+     * author, created, and slug are preserved from the existing page; the
+     * provider bumps the version; `editor` is the import actor; the search
+     * index, link graph, and render cache are updated in-band so the result
+     * is immediately searchable and link-resolvable.
+     */
+    private overwriteExistingPage;
+    /**
+     * A title the save door had to rewrite (#1455) is named in this file's
+     * warnings, so the import report says what the source called the page and
+     * what it is called here. Returns the title the page actually has.
+     */
+    private reportNormalisedTitle;
+    /**
+     * Create a new page from an import through PageManager save semantics
+     * (#880): the provider assigns versioning and the page comes out indexed,
+     * link-resolvable, and visible in Recent Changes — instead of the raw file
+     * write that left pages invisible to search until a manual reindex.
+     * `author` and `editor` are the import actor (unless the source frontmatter
+     * carries an author).
+     */
+    private createPageThroughPipeline;
+    /**
+     * Import a page from a URL
+     *
+     * Fetches the URL and imports it like a file import of the same HTML
+     * (#1337): the NCM normalizer (links, tables, `ncmVersion`), the #1332 fix
+     * steps, then a save through PageManager — validated, audited as the
+     * importer, versioned and indexed. The fetched page's schema.org metadata
+     * and the source URL are kept in frontmatter.
+     *
+     * @param url - URL to fetch and import
+     * @param options - The importer's context, and optional title and dryRun
+     * @returns Imported file info
+     */
+    importFromUrl(url: string, options: {
+        actorContext: ActorContext;
+        title?: string;
+        dryRun?: boolean;
+    }): Promise<ImportedFile>;
+    /**
+     * Run every Markdown fix step over an imported body (#1332), through
+     * PageManager — the steps Convert to NCM runs. Each step that changed
+     * something is added to `warnings` as a `converter-note`, which the import
+     * preview and the run notification already show. Without a PageManager the
+     * body is returned unchanged.
+     *
+     * @param body - Page body, without frontmatter
+     * @param warnings - The conversion's warnings, appended to
+     * @returns The fixed body
+     */
+    private applyFixSteps;
+    /** Style classes for up-converted tables (NCM §2.1), operator-configurable. */
+    private ncmTableClasses;
+    /**
+     * JSON/XML into a private store are files, not pages (#1389).
+     */
+    private importPrivateStoreFile;
+    /**
+     * Import attachments from a JSPWiki `-att/` directory alongside a page file.
+     *
+     * @param sourceFilePath - Path to the source `.txt` page file
+     * @param pageName - Decoded page name (used to link attachments)
+     * @param options - Import options (dryRun support)
+     * @returns Stats about imported attachments
+     */
+    private importPageAttachments;
+    /**
+     * Get MIME type from file extension
+     */
+    private getMimeType;
+    /**
+     * Get file extensions to process based on options
+     */
+    private getFileExtensions;
+    /**
+     * Find all files with matching extensions in directory
+     */
+    private findFiles;
+    /**
+     * Build YAML frontmatter from conversion result
+     */
+    private buildImportMetadata;
+    private buildFrontmatter;
+    /**
+     * Register new user-keywords to the custom configuration
+     *
+     * When importing JSPWiki pages with %%category%% blocks, this method
+     * adds extracted categories to the config so they appear in the
+     * user-keywords picker when editing pages.
+     *
+     * @param keywords - Array of keyword names to register
+     * @returns Number of new keywords added
+     */
+    private registerUserKeywordsToConfig;
+    /**
+     * Import a takeout into one of the requester's OWN private stores (#1472).
+     *
+     * The mirror of `PageManager.buildOwnStoreTakeout`, and not `importPages`:
+     * that reads a directory on the server, so a decrypted takeout would have to
+     * be staged on disk, and it converts every page, which would rewrite bodies
+     * that are already this system's own markdown. Here the archive is read in
+     * memory and each page and file goes through the same doors an edit and an
+     * upload use, so a sealed store encrypts on write with no new crypto.
+     *
+     * Refused before anything is written — never a partial import — when the
+     * requester is not the owner, the store is not theirs, or it is encrypted
+     * and locked in this session.
+     *
+     * Idempotent (operator, 2026-09-25): importing the same takeout twice leaves
+     * the store as the first import did.
+     *
+     *   - Files first, so page links can be pointed at them. The store's upload
+     *     finds a file whose bytes it already holds and returns that one, so a
+     *     second import stores nothing new.
+     *   - A page whose uuid the target store already holds is skipped:
+     *     `unchanged` when its body matches, `changed-since-takeout` when not —
+     *     the live page wins.
+     *   - A page whose uuid is used elsewhere on the site is skipped as
+     *     `uuid-elsewhere`, naming the page only when the requester may view it.
+     *   - A title held by a DIFFERENT page lands beside it as `Title (imported)`.
+     *
+     * This is not a restore: a takeout carries no history and no trash.
+     *
+     * @param ctx - Who is importing; only into their own container
+     */
+    importOwnStoreTakeout(ctx: ActorContext, options: {
+        store: string;
+        archive: Buffer;
+        limits: ZipReadLimits;
+        /** The uploaded file's name, for the report page. */
+        sourceName?: string;
+    }): Promise<TakeoutImportReport>;
+    /** The import report page's markdown. Private links are `[Title|store/Title]` (#1457). */
+    private takeoutReportMarkdown;
+    /**
+     * Where a uuid is already in use outside the target store, or null.
+     *
+     * The owner's other stores this session can read, then the public pages.
+     * `where` is set only for a page the requester may view: a uuid held by a
+     * page they cannot see is reported without naming it, so an import cannot
+     * be used to probe for pages.
+     */
+    private whereUuidLives;
+    /**
+     * Backup manager data (no persistent data to backup)
+     */
+    backup(): Promise<BackupData>;
+    /**
+     * Restore manager data (no persistent data to restore)
+     */
+    restore(backupData: BackupData): Promise<void>;
+    /**
+     * Shutdown the manager
+     */
+    shutdown(): Promise<void>;
+}
+export default ImportManager;

package/lib/src/managers/OrganizationManager.d.ts

@@ -1,4 +1,4 @@
-import BaseManager, { type BackupData, type ManagerBackupOptions, type ManagerRestoreOptions, type ManagerStats } from './BaseManager.js';
+import BaseManager, { type BackupFolder, type ManagerStats } from './BaseManager.js';
 import type { WikiEngine } from '../types/WikiEngine.js';
 import type { Organization, OrganizationInput, OrganizationUpdate } from '../types/Organization.js';
 import type { OrganizationProvider } from '../types/OrganizationProvider.js';
@@ -45,11 +45,13 @@ declare class OrganizationManager extends BaseManager {
     constructor(engine: WikiEngine);
     initialize(config?: Record<string, unknown>): Promise<void>;
     /** Provider accessor, mainly for tests. */
-    /** Organization records are in the backup (#1876), as `<stem>.organizations.tar` beside the document. */
-    backup(options?: ManagerBackupOptions): Promise<BackupData>;
-    /** Staged for the next start with the rest of a site restore (#1876); the live records are untouched. */
-    restore(backupData: BackupData, options?: ManagerRestoreOptions): Promise<void>;
-    restoreIsStaged(): boolean;
+    /** `ngdpbase.application.organization.storagedir`, resolved: where the records are. */
+    private configuredStorageDir;
+    /**
+     * Organization records are in the backup (#1876), as `<stem>.organizations.tar` beside the document. BackupManager backs it up, stages it in a site restore and puts it
+     * in place at the next start, before the records are read (#1985).
+     */
+    backupFolders(): BackupFolder[];
     getProvider(): OrganizationProvider | null;
     /**
      * Create the anchor organization from the instance's own identity (#1027).