e10-ebuilder-prototype 0.5.0

package/dist/store.js

@@ -0,0 +1,296 @@
+import { reservedWindowsName } from './runtime-support.mjs';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+import { randomUUID } from 'node:crypto';
+import { atomicJson, CaptureError, digest, fileDigest, mapLimit, id, readJson, safeName, VERSION, } from './common.js';
+import { defaults, } from './model.js';
+import { temporarySummary } from './temporary-records.js';
+import { formResult, verifiedForm } from './forms.js';
+import { navigationKey } from './menus.js';
+export const htmlArtifact = (key, kind = 'page') => kind === 'form' ? `form-html/${navigationKey(key)}.html` : `${id(key)}.html`;
+export class Store {
+    root;
+    meta;
+    artifacts;
+    constructor(dir) {
+        this.root = path.resolve(dir);
+        // Stable on-disk format keeps existing tasks resumable after the rename.
+        this.meta = path.join(this.root, '.e10-page-capture');
+        this.artifacts = path.join(this.root, 'artifacts');
+    }
+    async load() {
+        const s = await readJson(path.join(this.meta, 'task.json'));
+        if (s.schema !== 1 || !s.settings || !/^\d+$/.test(s.appId))
+            throw new CaptureError('STATE_INVALID', '任务状态格式无效');
+        return s;
+    }
+    async save(s) {
+        await atomicJson(path.join(this.meta, 'task.json'), s);
+    }
+    async init(appId, settings = {}) {
+        try {
+            const s = await this.load();
+            if (s.appId !== appId)
+                throw new CaptureError('TASK_MISMATCH', '目录已绑定另一个 appId');
+            if (Object.entries(settings).some(([k, v]) => s.settings[k] !== v))
+                throw new CaptureError('SETTINGS_MISMATCH', '已有任务参数不同，请使用新目录');
+            return s;
+        }
+        catch (e) {
+            if (e.code !== 'ENOENT')
+                throw e;
+        }
+        const s = {
+            schema: 1,
+            version: VERSION,
+            appId: id(appId, 'appId'),
+            createdAt: new Date().toISOString(),
+            htmlRequired: true,
+            siteRequired: true,
+            menuRequired: true,
+            settings: { ...defaults, ...settings },
+        };
+        await this.save(s);
+        return s;
+    }
+    bind(s, a) {
+        if (s.requestedOrigin && new URL(a.baseUrl).origin !== s.requestedOrigin)
+            throw new CaptureError('ENVIRONMENT_MISMATCH', '应用地址与当前登录环境不一致，请使用对应环境的登录 Profile');
+        const environment = {
+            origin: new URL(a.baseUrl).origin,
+            identityHash: digest(`${a.userId}\0${a.tenantKey}`),
+        };
+        if (s.environment && JSON.stringify(environment) !== JSON.stringify(s.environment))
+            throw new CaptureError('ENVIRONMENT_MISMATCH', '当前登录环境/用户/租户与任务不符，请切换原 Profile 或使用新目录');
+        s.environment = environment;
+    }
+    receiptPath(pageId) {
+        return path.join(this.meta, 'results', `${id(pageId)}.json`);
+    }
+    async result(pageId) {
+        try {
+            return await readJson(this.receiptPath(pageId));
+        }
+        catch (e) {
+            if (e.code === 'ENOENT')
+                return undefined;
+            throw e;
+        }
+    }
+    async saveResult(r) {
+        await atomicJson(this.receiptPath(r.id), r);
+    }
+    htmlReceiptPath(pageId, kind = 'page') {
+        return path.join(this.meta, kind === 'form' ? 'form-html-results' : 'html-results', `${kind === 'form' ? navigationKey(pageId) : id(pageId)}.json`);
+    }
+    async htmlResult(pageId, kind = 'page') {
+        try {
+            return await readJson(this.htmlReceiptPath(pageId, kind));
+        }
+        catch (e) {
+            if (e.code === 'ENOENT')
+                return undefined;
+            throw e;
+        }
+    }
+    async saveHtmlResult(r) {
+        await atomicJson(this.htmlReceiptPath(r.id, r.kind), r);
+    }
+    async verifiedHtml(r, png) {
+        if (r?.status !== 'succeeded' ||
+            !r.file ||
+            !r.sha256 ||
+            r.id !== png?.id ||
+            r.file !== htmlArtifact(png.id, r.kind) ||
+            r.sourceSha256 !== png?.sha256 ||
+            png?.status !== 'succeeded')
+            return false;
+        try {
+            return (await fileDigest(this.artifact(r.file))) === r.sha256;
+        }
+        catch {
+            return false;
+        }
+    }
+    artifact(relative) {
+        const p = path.resolve(this.artifacts, relative);
+        if (!p.startsWith(this.artifacts + path.sep))
+            throw new CaptureError('PATH_ESCAPE', '文件路径超出产物目录');
+        return p;
+    }
+    async verified(r) {
+        if (r?.status !== 'succeeded' || !r.file || !r.sha256)
+            return false;
+        try {
+            const filename = this.artifact(r.file), handle = await fs.open(filename, 'r');
+            let valid = false;
+            try {
+                const header = Buffer.alloc(8);
+                const result = await handle.read(header, 0, 8, 0);
+                valid =
+                    (await handle.stat()).size === r.bytes &&
+                        result.bytesRead === 8 &&
+                        header.equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]));
+            }
+            finally {
+                await handle.close();
+            }
+            return valid && (await fileDigest(filename)) === r.sha256;
+        }
+        catch {
+            return false;
+        }
+    }
+    async status(s) {
+        const results = await mapLimit(s.pages || [], 4, (p) => this.result(p.id));
+        const valid = await mapLimit(results, 4, (r) => this.verified(r));
+        const succeeded = valid.filter(Boolean).length;
+        const pending = results.filter((r, i) => !r || r.status === 'running' || (r.status === 'succeeded' && !valid[i])).length;
+        const failed = results.filter((r) => r?.status === 'failed').length;
+        const htmlResults = await mapLimit(s.pages || [], 4, (p) => this.htmlResult(p.id));
+        const htmlValid = await mapLimit(htmlResults, 4, async (r) => {
+            const index = s.pages.findIndex((p) => p.id === r?.id);
+            return index >= 0 && valid[index] && (await this.verifiedHtml(r, results[index]));
+        });
+        const html = {
+            required: !!s.htmlRequired,
+            succeeded: htmlValid.filter(Boolean).length,
+            failed: htmlResults.filter((r, i) => valid[i] && r?.status === 'failed' && r.sourceSha256 === results[i]?.sha256).length,
+            running: htmlResults.filter((r, i) => valid[i] && r?.status === 'running' && r.sourceSha256 === results[i]?.sha256).length,
+            pending: 0,
+        };
+        html.pending = s.htmlRequired ? succeeded - html.succeeded - html.failed : 0;
+        const pageHtml = { ...html };
+        const formResults = await mapLimit(s.menuRequired ? s.formPages || [] : [], 4, (p) => formResult(this, p.id));
+        const formValid = await mapLimit(formResults, 4, (r) => verifiedForm(this, r));
+        const formHtmlResults = await mapLimit(s.menuRequired ? s.formPages || [] : [], 4, (p) => this.htmlResult(p.id, 'form'));
+        const formHtmlValid = await mapLimit(formHtmlResults, 4, (r) => {
+            const i = s.formPages.findIndex((p) => p.id === r?.id);
+            return i >= 0 && formValid[i] ? this.verifiedHtml(r, formResults[i]) : Promise.resolve(false);
+        });
+        const collection = {
+            succeeded: formValid.filter(Boolean).length,
+            failed: formResults.filter((r) => r?.status === 'failed').length,
+            pending: formResults.filter((r, i) => !r || r.status === 'running' || (r.status === 'succeeded' && !formValid[i])).length,
+        };
+        const formHtml = {
+            succeeded: formHtmlValid.filter(Boolean).length,
+            failed: formHtmlResults.filter((r, i) => formValid[i] && r?.status === 'failed' && r.sourceSha256 === formResults[i]?.sha256).length,
+            running: formHtmlResults.filter((r, i) => formValid[i] && r?.status === 'running' && r.sourceSha256 === formResults[i]?.sha256).length,
+            pending: 0,
+        };
+        formHtml.pending = collection.succeeded - formHtml.succeeded - formHtml.failed;
+        for (const key of ['succeeded', 'failed', 'running', 'pending'])
+            html[key] += formHtml[key];
+        let state = !s.pages ||
+            (s.siteRequired && (!s.forms || !s.appName)) ||
+            (s.menuRequired && (!s.menus || !s.formPages))
+            ? 'DISCOVER'
+            : collection.pending
+                ? 'COLLECT'
+                : pending
+                    ? 'CAPTURE'
+                    : html.pending
+                        ? 'HTML'
+                        : 'PACKAGE';
+        const temporaryRecords = await temporarySummary(this);
+        if (temporaryRecords.pending && state !== 'DISCOVER')
+            state = 'COLLECT';
+        const partial = !!(failed || collection.failed || (s.htmlRequired && html.failed));
+        if (s.archive && state === 'PACKAGE' && s.archive.partial === partial) {
+            try {
+                const siteValid = !s.siteRequired ||
+                    (s.archive.site &&
+                        (await mapLimit(s.archive.site.files, 4, async (f) => (await fileDigest(this.artifact(f.path))) === f.sha256)).every(Boolean));
+                if (siteValid && (await fileDigest(this.artifact(s.archive.path))) === s.archive.sha256)
+                    state = partial ? 'PARTIAL' : 'DONE';
+            }
+            catch {
+                /* missing archive requires packaging */
+            }
+        }
+        return {
+            state,
+            total: s.pages?.length ?? 0,
+            succeeded,
+            failed,
+            pending,
+            results,
+            htmlResults,
+            formResults,
+            formHtmlResults,
+            pageHtml,
+            formHtml,
+            collection,
+            temporaryRecords,
+            html,
+            appName: s.appName,
+            forms: s.menuRequired
+                ? {
+                    total: s.formPages?.length ?? 0,
+                    integrated: formHtml.succeeded,
+                    placeholders: collection.failed + formHtml.failed,
+                    collection,
+                    html: formHtml,
+                }
+                : { total: s.forms?.length ?? 0, integrated: 0, placeholders: s.forms?.length ?? 0 },
+            entry: (state === 'DONE' || state === 'PARTIAL') && s.archive?.site
+                ? this.artifact(s.archive.site.entry)
+                : undefined,
+            archive: state === 'DONE' || state === 'PARTIAL' ? this.artifact(s.archive.path) : undefined,
+        };
+    }
+    async lock(action) {
+        await fs.mkdir(this.meta, { recursive: true });
+        const filename = path.join(this.meta, 'lock.json');
+        const token = randomUUID();
+        for (let i = 0; i < 2; i++) {
+            try {
+                await fs.writeFile(filename, JSON.stringify({ pid: process.pid, host: os.hostname(), token }), { flag: 'wx', mode: 0o600 });
+                break;
+            }
+            catch (e) {
+                if (e.code !== 'EEXIST')
+                    throw e;
+                const owner = await readJson(filename).catch(() => null);
+                let alive = true;
+                if (owner?.host === os.hostname()) {
+                    try {
+                        process.kill(owner.pid, 0);
+                    }
+                    catch (err) {
+                        if (err.code === 'ESRCH')
+                            alive = false;
+                    }
+                }
+                if (alive || i === 1)
+                    throw new CaptureError('TASK_LOCKED', '另一个进程持有任务锁；不要并行操作同一目录', owner);
+                await fs.rm(filename);
+            }
+        }
+        try {
+            return await action();
+        }
+        finally {
+            const owner = await readJson(filename).catch(() => null);
+            if (owner?.token === token)
+                await fs.rm(filename, { force: true });
+        }
+    }
+}
+export function filenames(pages) {
+    const names = pages.map((p) => {
+        const name = safeName(p.name);
+        return reservedWindowsName(name) ? `_${name}` : name;
+    });
+    const keys = names.map((n) => n.toLocaleLowerCase('en-US'));
+    const used = new Set();
+    return new Map(pages.map((p, i) => {
+        let n = keys.filter((v) => v === keys[i]).length > 1 ? `${names[i]}-${p.id}` : names[i];
+        while (used.has(n.toLowerCase()))
+            n += `-${p.id}`;
+        used.add(n.toLowerCase());
+        return [p.id, `${n}.png`];
+    }));
+}