@josfox/jos 4.1.0 → 5.0.0

package/src/commands/migrate.js

@@ -0,0 +1,214 @@
+/**
+ * JOS MIGRATE - Reseal a 0.0.7 document sealed by security.integrity_ref as a 2.0 document sealed by integrity.hash
+ * (PLAN-INTEGRITY-REF-0.0.7 §5, decision D7: a host tool, not a conformance requirement).
+ *
+ * The file is read once. jos-core's verifyLegacyRef (the one check `run` uses) verifies the manifest against those
+ * octets; a document that does not verify is exit 3 and nothing is written, so a changed document cannot be laundered
+ * into a valid seal. The migrated document is jos-core's normalization with three changes: security.integrity_ref
+ * (x-legacy-integrity_ref after normalization) removed, security.permissions moved to security.x-legacy-permissions
+ * (4.1.0 never applied it; 2.0 would apply an empty list as a ceiling with no shell), and integrity.hash =
+ * "sha256:" + the payload of its art_ (SPEC-2.0 §8.1). Before writing, the exact octets it writes, read back through
+ * jos-core's own reader (parseText), must be that document and validate as 2.0 with that art_ (rc.8, F-05: an object
+ * can be valid in memory while its JSON text is not), and its dry run must give the code and plan of the original's. Exit 0 written (or already 2.0), 1 not written (exists,
+ * cannot write, self-check), 2 invalid, 3 does not verify, 5 no kernel, 64 usage or not a sealed 0.0.7 document.
+ */
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const kernel = require('../kernel');
+const ui = require('../ui');
+const { RULE } = ui;
+const { parse, problem } = require('../args');
+
+const OPTIONS = { '--out': true, '--in-place': false, '--force': false };
+const USAGE = 'usage: jos migrate <doc.jos> [--out <file> | --in-place] [--force]';
+
+const HELP = (C) => `
+${C.purple}${RULE}${C.reset}
+${C.bold}JOS MIGRATE${C.reset} // Reseal a 0.0.7 document as 2.0
+${C.purple}${RULE}${C.reset}
+
+${C.bold}Usage:${C.reset}
+  jos migrate <doc.jos> [--out <file> | --in-place] [--force]
+
+  For a 0.0.7 document sealed by security.integrity_ref (what jos init of 4.1.0
+  wrote, with its .sig.json manifest). The manifest is verified first, exactly as
+  jos run verifies it: a document that does not match is refused (exit 3) and
+  nothing is written. Then it writes the 2.0 document jos-core normalizes, sealed
+  by integrity.hash instead of the manifest:
+
+    security.integrity_ref        removed (the manifest is no longer read)
+    security.permissions          moved to security.x-legacy-permissions: 4.1.0
+                                  never applied it, and 2.0 would apply an empty
+                                  list as "no shell"
+    integrity.hash                sha256 of the migrated document (SPEC-2.0 8.1)
+
+  The result must validate and plan the same steps as the original (dry run)
+  before it is written. The input and its manifest are left as they are.
+
+${C.bold}Options:${C.reset}
+  --out <file>   Where to write (default: <name>.v2.jos beside the input)
+  --in-place     Replace the input itself (the manifest file stays, unused)
+  --force        Overwrite an existing --out or <name>.v2.jos (never the input)
+
+${C.bold}Exit codes:${C.reset}
+  0 written, or already 2.0 (nothing to do); 1 not written (the output exists,
+  cannot write, self-check failed); 2 invalid document; 3 integrity_ref does not
+  verify; 5 no kernel; 64 usage, or not a 0.0.7 document with integrity_ref
+
+  1.1.0 and 1.0.0 documents are not migrated in this version: they run as they are.
+
+${C.bold}Example:${C.reset}
+  jos migrate ref.jos            # writes ref.v2.jos
+  jos run ref.v2.jos
+`;
+
+const plain = (s) => (s.length > 1 && s.startsWith('-') ? './' + s : s);
+const canonical = (core, v) => core.canonicalize(v === undefined ? null : v);
+
+// <dir>/<name>.v2.jos for <dir>/<name>.jos or .jos.json; any other name gets .v2.jos appended
+function defaultOut(file) {
+    return path.join(path.dirname(file), path.basename(file).replace(/\.jos(\.json)?$/, '') + '.v2.jos');
+}
+
+function sameFile(a, b) {
+    try { return fs.realpathSync(a) === fs.realpathSync(b); } catch (e) { return path.resolve(a) === path.resolve(b); }
+}
+
+// The 2.0 document migrate writes, from jos-core's normalization; returns { doc, artifact, moved }.
+function migrated(core, normalized) {
+    const m = JSON.parse(JSON.stringify(normalized));
+    const sec = m.security;
+    delete sec['x-legacy-integrity_ref'];
+    const moved = Object.prototype.hasOwnProperty.call(sec, 'permissions');
+    if (moved) {
+        sec['x-legacy-permissions'] = sec.permissions;
+        delete sec.permissions;
+    }
+    m.integrity = { hash: '' };
+    const artifact = core.artifactId(m);
+    m.integrity.hash = 'sha256:' + core.josid.parse(artifact).payload;
+    return { doc: m, artifact, moved };
+}
+
+exports.execute = async (args) => {
+    const err = ui.printer(process.stderr);
+    const { C } = err;
+    const { opts, files, help, error } = parse(args, OPTIONS);
+    if (help) {
+        const out = ui.printer(process.stdout);
+        out.text(HELP(out.C));
+        return 0;
+    }
+    if (error || files.length !== 1 || (opts['--in-place'] && opts['--out'] !== undefined)) {
+        err.fail('jos migrate: ' + (error ? problem(error) : files.length !== 1 ? 'takes exactly one file' : '--out and --in-place exclude each other'));
+        err.hint(USAGE);
+        return 64;
+    }
+
+    let k;
+    let core;
+    try { k = kernel.locate(); core = kernel.need(k, ['loadFileBytes', 'parseText', 'verifyLegacyRef', 'artifactId', 'canonicalize', 'josid.parse']); } catch (e) { return kernel.fail(e); }
+    const file = path.resolve(files[0]);
+    const shown = files[0];
+    const dir = path.dirname(file);
+
+    let doc;
+    let bytes;
+    let v;
+    try {
+        ({ doc, bytes } = core.loadFileBytes(file));
+        v = core.validate(doc);
+    } catch (e) {
+        if (!(e instanceof core.JosError)) throw e;
+        v = { valid: false, errors: e.errors };
+    }
+    if (!v.valid) {
+        err.fail('jos migrate: ' + shown + ' is not a valid document (code 2)');
+        for (const e of v.errors) err.hint(e);
+        return 2;
+    }
+    if (v.form === '2.0') {
+        err.ok(shown + ' is already 2.0', '(' + v.artifact + '): nothing to migrate');
+        return 0;
+    }
+    const ref = v.form === '0.0.7' ? core.verifyLegacyRef(doc, bytes, dir) : null;
+    if (ref === null) {
+        const what = v.form === '0.0.7' ? 'a 0.0.7 document without security.integrity_ref' : 'a ' + v.input_version + ' document';
+        err.fail('jos migrate: ' + shown + ' is ' + what + ': nothing to reseal');
+        err.hint('jos migrate rewrites 0.0.7 documents sealed by security.integrity_ref (in this version);');
+        err.hint('this one runs as it is, normalized in memory: jos run ' + ui.shellArg(plain(shown)));
+        return 64;
+    }
+    if (!ref.ok) {
+        err.fail('jos migrate: ' + shown + ': ' + ref.reason, '(code ' + ref.code + ')');
+        err.hint('nothing written: a document that does not match its manifest is not resealed');
+        return ref.code;
+    }
+
+    const out = opts['--in-place'] ? file : opts['--out'] !== undefined ? path.resolve(opts['--out']) : defaultOut(file);
+    const outShown = opts['--in-place'] ? shown : opts['--out'] !== undefined ? opts['--out'] : path.join(path.dirname(shown), path.basename(out));
+    if (!opts['--in-place'] && sameFile(out, file)) {
+        err.fail('jos migrate: ' + outShown + ' is the input', '(use --in-place to replace it)');
+        return 1;
+    }
+    if (!opts['--in-place'] && fs.existsSync(out) && !opts['--force']) {
+        err.fail('jos migrate: ' + outShown + ' exists', '(use --force to overwrite)');
+        return 1;
+    }
+
+    // Self-check before anything is written (also with --in-place): the octets it writes, read back by the kernel's own
+    // reader, are the migrated document, a valid 2.0 document with the art_ it is sealed with (rc.8, F-05: the object in
+    // memory is not what a later run reads), and its dry run (seal verified) gives the code and the plan of the
+    // original's dry run (manifest verified).
+    const m = migrated(core, v.document);
+    const octets = Buffer.from(JSON.stringify(m.doc, null, 2) + '\n', 'utf8');
+    let read = null;
+    let v2;
+    try {
+        read = core.parseText(octets);
+        v2 = core.validate(read);
+    } catch (e) {
+        if (!(e instanceof core.JosError)) throw e;
+        v2 = { valid: false, errors: e.errors };
+    }
+    const same = read !== null && canonical(core, read) === canonical(core, m.doc);
+    const env = process.env;
+    const before = await core.run(doc, { cwd: dir, dir, source: bytes, env, dryRun: true });
+    const after = await core.run(read === null ? m.doc : read, { cwd: dir, dir, env, dryRun: true });
+    if (!v2.valid || !same || v2.form !== '2.0' || v2.artifact !== m.artifact || after.code !== before.code || canonical(core, after.plan) !== canonical(core, before.plan)) {
+        const why = !v2.valid ? 'validate' : !same ? 'read back from the octets it would write as the document it computed' : 'plan what the original plans';
+        err.fail('jos migrate: self-check failed: the migrated document does not ' + why, '(nothing written)');
+        err.hint('original dry run: code ' + before.code + (before.reason ? ' ' + before.reason : '') + '; migrated: code ' + after.code + (after.reason ? ' ' + after.reason : '') + (v2.valid ? '' : '; ' + v2.errors.join('; ')));
+        return 1;
+    }
+
+    const tmp = out + '.tmp';
+    try {
+        fs.writeFileSync(tmp, octets);
+        fs.renameSync(tmp, out);
+    } catch (e) {
+        try { fs.rmSync(tmp, { force: true }); } catch (e2) { /* nothing to clean */ }
+        err.fail('jos migrate: cannot write ' + outShown + ': ' + (e && e.code ? e.code : e.message));
+        return 1;
+    }
+
+    err.header('JOS MIGRATE', 'jos-core ' + k.version);
+    err.field('Input', shown + ` ${C.gray}(0.0.7, ${v.artifact})${C.reset}`);
+    err.ok('integrity_ref verified', '(' + ref.manifest + ', sha256 ' + ref.sha256.slice(0, 12) + '…, ' + ref.eol + ')');
+    err.ok('Written: ' + outShown, '(2.0.0 ' + m.doc.jos.kind + ', ' + m.artifact + ')');
+    err.line();
+    err.line(`${C.bold}Changes:${C.reset}`);
+    err.item('normalized to 2.0.0 by jos-core; the 0.0.7 members it does not map are kept as x-legacy-*');
+    err.item('security.integrity_ref removed: ' + ref.manifest + ' is no longer read');
+    if (m.moved) err.item('security.permissions moved to security.x-legacy-permissions (4.1.0 never applied it; in 2.0 an empty list means no shell)');
+    err.item('integrity.hash = ' + m.doc.integrity.hash.slice(0, 19) + '… (SPEC-2.0 8.1): any change to the document is now code 3');
+    err.ok('Self-check', '(the written octets read back as valid 2.0, same art_ as its seal, the dry run plans what the original plans)');
+    if (opts['--in-place']) err.hint('The input was replaced; ' + ref.manifest + ' is no longer used by it.');
+    err.hint('Next: jos run ' + ui.shellArg(plain(outShown)));
+    err.line();
+    return 0;
+};
+
+exports.defaultOut = defaultOut;
+exports.migrated = migrated;

package/src/commands/overview.js

@@ -0,0 +1,118 @@
+/**
+ * `jos` with no arguments: what a .jos is and what jos does, in six lines, then a read-only look at the current
+ * folder: the .jos / .jos.json documents in it and one folder down, and the folders with a .jos/manifest.jos, each
+ * with the commands to validate and run it. Nothing runs and nothing is written. The home folder (or a folder that
+ * contains it, such as /) is never scanned. Exit 0.
+ */
+'use strict';
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const pkg = require('../../package.json');
+const kernel = require('../kernel');
+const ui = require('../ui');
+
+const DOC = /\.jos(\.json)?$/;
+const MAX_SHOWN = 12;
+const MAX_DIRS = 200;
+const SKIP_DIRS = new Set(['node_modules']);
+
+const isFile = (p) => { try { return fs.statSync(p).isFile(); } catch (e) { return false; } };
+// The entries of a folder sorted by name (code unit order), or none when it cannot be read.
+const list = (d) => { try { return fs.readdirSync(d, { withFileTypes: true }).sort((a, b) => (a.name > b.name) - (a.name < b.name)); } catch (e) { return []; } };
+
+// What is in `cwd` and one level down: { docs: [relative paths], folders: [relative dirs with .jos/manifest.jos] }.
+function discover(cwd) {
+    const docs = [];
+    const folders = [];
+    if (isFile(path.join(cwd, '.jos', 'manifest.jos'))) folders.push('.');
+    const top = list(cwd);
+    const subdirs = [];
+    for (const e of top) {
+        if (e.isFile() && DOC.test(e.name)) docs.push(e.name);
+        else if (e.isDirectory() && !e.name.startsWith('.') && !SKIP_DIRS.has(e.name)) subdirs.push(e.name);
+    }
+    for (const d of subdirs.slice(0, MAX_DIRS)) {
+        if (isFile(path.join(cwd, d, '.jos', 'manifest.jos'))) folders.push(d);
+        for (const e of list(path.join(cwd, d))) {
+            if (e.isFile() && DOC.test(e.name)) docs.push(d + '/' + e.name);
+        }
+    }
+    return { docs, folders };
+}
+
+// True when scanning `cwd` would mean scanning the home folder: cwd is home, or contains it (/, /Users).
+function isHomeOrAbove(cwd, home) {
+    const rel = path.relative(cwd, home);
+    return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel));
+}
+
+// "atom hello" / "invalid: <first error>" for a document, read and validated (never run) by jos-core.
+function label(core, file) {
+    try {
+        const v = core.validate(core.loadFile(file));
+        if (!v.valid) return 'invalid: ' + v.errors[0];
+        return v.document.jos.kind + (typeof v.document.id === 'string' ? ' ' + v.document.id : '') + ', valid';
+    } catch (e) {
+        if (!(e instanceof core.JosError)) throw e;
+        return 'invalid: ' + e.errors[0];
+    }
+}
+
+exports.execute = async () => {
+    const out = ui.printer(process.stdout);
+    const { C } = out;
+    let k = null;
+    try { k = kernel.locate(); } catch (e) { if (!(e instanceof kernel.KernelNotFound)) throw e; }
+    out.header(`${C.cyan}JOS${C.reset} ${C.bold}v${pkg.version}`, k ? 'host of jos-core ' + k.version : 'jos-core not found (run and validate exit 5)');
+    out.line('  A .jos is an intention you declare: what to do, when it is done, what to avoid.');
+    out.item('Validated without running anything               ' + `${C.cyan}jos validate <file>${C.reset}`);
+    out.item('Run with guarantees: limits, criteria, receipts   ' + `${C.cyan}jos run <file>${C.reset}`);
+    out.item('Hash-chained ledger of every run, verifiable      ' + `${C.cyan}jos runs${C.reset}, ${C.cyan}jos ledger verify <run>${C.reset}`);
+    out.item('Content identity: art_ (content hash), JOSID      ' + `${C.cyan}jos ids <file>${C.reset}`);
+    out.item('Fractal: atom, flow, a folder with .jos/          ' + `${C.cyan}jos run <folder>${C.reset}`);
+    out.item('Reusable: use, templates, digests                 ' + `${C.cyan}jos run <flow> --library <dir>${C.reset}`);
+    out.hint('Two reference kernels (JavaScript, Python) give identical results. jos itself opens no network connection.');
+    out.line();
+
+    const cwd = process.cwd();
+    if (isHomeOrAbove(cwd, os.homedir())) {
+        out.line(`${C.bold}This folder:${C.reset} ${cwd}`);
+        out.hint('Not scanned: it is your home folder or contains it. cd into a project folder, or:');
+        out.cmd('jos start');
+        out.line();
+        return 0;
+    }
+    const found = discover(cwd);
+    out.line(`${C.bold}This folder:${C.reset} ${cwd} ${C.gray}(read only; nothing runs until you type a command)${C.reset}`);
+    if (!found.docs.length && !found.folders.length) {
+        out.hint('No .jos documents here or one folder down. Write four examples to start with:');
+        out.cmd('jos start');
+        out.line();
+        out.hint('All commands: jos --help');
+        out.line();
+        return 0;
+    }
+    const entries = found.folders.map((d) => ({ kind: 'folder', rel: d })).concat(found.docs.map((f) => ({ kind: 'doc', rel: f })));
+    for (const e of entries.slice(0, MAX_SHOWN)) {
+        if (e.kind === 'folder') {
+            const manifest = (e.rel === '.' ? '' : e.rel + '/') + '.jos/manifest.jos';
+            out.item((e.rel === '.' ? 'this folder' : e.rel + '/') + ` ${C.gray}(a .jos/ folder` + (k ? ': ' + label(k.core, path.join(cwd, manifest)) : '') + `)${C.reset}`);
+            out.cmd('jos validate ' + ui.shellArg(manifest));
+            out.cmd('jos run ' + ui.shellArg(e.rel));
+        } else {
+            out.item(e.rel + (k ? ` ${C.gray}(${label(k.core, path.join(cwd, e.rel))})${C.reset}` : ''));
+            const arg = (e.rel.startsWith('-') ? '-- ' : '') + ui.shellArg(e.rel);
+            out.cmd('jos validate ' + arg);
+            out.cmd('jos run ' + arg);
+        }
+    }
+    if (entries.length > MAX_SHOWN) out.hint('… and ' + (entries.length - MAX_SHOWN) + ' more');
+    out.line();
+    out.hint('All commands: jos --help');
+    out.line();
+    return 0;
+};
+
+module.exports.discover = discover;
+module.exports.isHomeOrAbove = isHomeOrAbove;