@greenpandastudios/aug-cli 0.20.1 → 0.21.0

package/src/native-artifacts.js

@@ -0,0 +1,207 @@
+import { createHash } from 'node:crypto';
+import { chmodSync, closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { UnpackSync } from 'tar';
+import { withPackageLockAsync } from "./package-locking.js";
+import { nativePath, nativeHostTarget, selectNativeArtifact } from "./native-contracts.js";
+import { compilerVersion, readPackage, sourcePaths, projectPackages } from "./package-manager.js";
+import { sourceAlias } from "./git-packages.js";
+const sha = (bytes) => createHash('sha256').update(bytes).digest('hex');
+const files = (directory, prefix = '') => readdirSync(join(directory, prefix), { withFileTypes: true }).flatMap(entry => {
+    const name = prefix ? prefix + '/' + entry.name : entry.name;
+    if (entry.isSymbolicLink())
+        throw new Error('NATIVE_INTEGRITY: Native archives cannot contain symbolic links');
+    return entry.isDirectory() ? files(directory, name) : entry.isFile() ? [name] : (() => { throw new Error('NATIVE_INTEGRITY: Native archives require regular files'); })();
+});
+/** Verify the complete extracted file set. Hashes never authorize arbitrary install scripts. */
+export function verifyArtifactFiles(directory, archive) {
+    const path = nativePath(archive.fileManifest), manifest = JSON.parse(readFileSync(join(directory, path), 'utf8'));
+    if (manifest.format !== 1 || !manifest.files || Array.isArray(manifest.files) || typeof manifest.files !== 'object')
+        throw new Error('NATIVE_INTEGRITY: Invalid artifact file manifest');
+    const expected = manifest.files, actual = files(directory).sort();
+    if (JSON.stringify(actual) !== JSON.stringify([...Object.keys(expected), path].sort()))
+        throw new Error('NATIVE_INTEGRITY: Artifact file set differs from its manifest');
+    for (const [file, digest] of Object.entries(expected)) {
+        nativePath(file);
+        if (!/^[0-9a-f]{64}$/.test(digest) || !lstatSync(join(directory, file)).isFile() || sha(readFileSync(join(directory, file))) !== digest)
+            throw new Error('NATIVE_INTEGRITY: Artifact file hash mismatch: ' + file);
+    }
+    return expected;
+}
+/** Bounded HTTPS downloads with checked redirects. No external downloader/toolchain is used. */
+export async function downloadVerified(archive) {
+    if (!/^[0-9a-f]{64}$/.test(archive.sha256) || !Number.isSafeInteger(archive.maximumDownloadBytes) || archive.maximumDownloadBytes < 1)
+        throw new Error('NATIVE_INTEGRITY: Invalid archive digest or size bound');
+    let url = new URL(archive.url), response;
+    const signal = AbortSignal.timeout(120000);
+    for (let i = 0; i <= 5; i++) {
+        if (url.protocol !== 'https:' || url.username || url.password || url.hash)
+            throw new Error('NATIVE_INTEGRITY: Artifact downloads and redirects require HTTPS without credentials');
+        response = await fetch(url, { redirect: 'manual', signal });
+        if ([301, 302, 303, 307, 308].includes(response.status)) {
+            const location = response.headers.get('location');
+            await response.body?.cancel();
+            if (!location || i === 5)
+                throw new Error('NATIVE_DOWNLOAD: Too many or invalid redirects');
+            url = new URL(location, url);
+            continue;
+        }
+        break;
+    }
+    if (!response?.ok || !response.body)
+        throw new Error(`NATIVE_DOWNLOAD: Missing artifact (${response?.status}) at ${archive.url}. Ask the package maintainer to publish the matching artifact. No source build was started.`);
+    if (Number(response.headers.get('content-length') ?? 0) > archive.maximumDownloadBytes) {
+        await response.body.cancel();
+        throw new Error('NATIVE_INTEGRITY: Artifact exceeds its download limit');
+    }
+    const chunks = [];
+    let length = 0;
+    for await (const chunk of response.body) {
+        length += chunk.length;
+        if (length > archive.maximumDownloadBytes)
+            throw new Error('NATIVE_INTEGRITY: Artifact exceeds its download limit');
+        chunks.push(Buffer.from(chunk));
+    }
+    const bytes = Buffer.concat(chunks);
+    if (sha(bytes) !== archive.sha256)
+        throw new Error('NATIVE_INTEGRITY: Downloaded artifact SHA-256 does not match the locked package');
+    return bytes;
+}
+/** Content-addressed immutable installs; failed extraction never installs an accepted cache. */
+export async function ensureVerifiedArchive(archive, options = {}) {
+    if (!/^[0-9a-f]{64}$/.test(archive.sha256) || !Number.isSafeInteger(archive.maximumUnpackedBytes) || archive.maximumUnpackedBytes < 1)
+        throw new Error('NATIVE_INTEGRITY: Invalid archive digest or unpacked size bound');
+    nativePath(archive.fileManifest);
+    const cache = resolve(options.cache ?? process.env.AUG_NATIVE_ARTIFACT_CACHE ?? join(homedir(), '.cache/augscript/native-artifacts'));
+    mkdirSync(cache, { recursive: true });
+    const destination = join(cache, archive.sha256);
+    return withPackageLockAsync(destination + '.lock', async () => {
+        if (existsSync(destination)) {
+            if (lstatSync(destination).isSymbolicLink())
+                throw new Error('NATIVE_INTEGRITY: Artifact cache cannot be a symbolic link');
+            verifyArtifactFiles(destination, archive);
+            return destination;
+        }
+        if (options.offline)
+            throw new Error('NATIVE_OFFLINE: Locked native artifact is not cached. Run aug install online once. ' + archive.url);
+        const stage = mkdtempSync(join(cache, '.install-')), transport = join(stage, 'download.tar.gz'), output = join(stage, 'files');
+        mkdirSync(output);
+        try {
+            writeFileSync(transport, await downloadVerified(archive));
+            const seen = new Set();
+            let unpacked = 0;
+            const extractor = new UnpackSync({ cwd: output, strict: true, preserveOwner: false, filter: (path, entry) => {
+                    const name = path.replace(/\/$/, '');
+                    nativePath(name);
+                    if (!('type' in entry))
+                        throw new Error('NATIVE_INTEGRITY: Expected an archive entry');
+                    if (!['File', 'Directory'].includes(entry.type))
+                        throw new Error('NATIVE_INTEGRITY: Archive links and special files are forbidden');
+                    if (seen.has(name))
+                        throw new Error('NATIVE_INTEGRITY: Duplicate archive path ' + name);
+                    seen.add(name);
+                    unpacked += entry.size;
+                    if (seen.size > 20000 || unpacked > archive.maximumUnpackedBytes)
+                        throw new Error('NATIVE_INTEGRITY: Archive exceeds its unpacked size or file limit');
+                    return true;
+                } });
+            // tar's synchronous file convenience API has no error listener. A disk
+            // write failure can otherwise become an uncaught stream event instead of
+            // the CLI's actionable error and rejected-install cleanup.
+            let extractionError;
+            extractor.on('error', (error) => { extractionError ??= error; });
+            const input = openSync(transport, 'r');
+            try {
+                let length;
+                do {
+                    const buffer = Buffer.allocUnsafe(16 * 1024 * 1024);
+                    length = readSync(input, buffer, 0, buffer.length, null);
+                    if (length)
+                        extractor.write(buffer.subarray(0, length));
+                    if (extractionError)
+                        throw extractionError;
+                } while (length);
+                extractor.end();
+                if (extractionError)
+                    throw extractionError;
+            }
+            finally {
+                closeSync(input);
+            }
+            verifyArtifactFiles(output, archive);
+            for (const executable of options.executables ?? []) {
+                nativePath(executable);
+                if (!lstatSync(join(output, executable)).isFile())
+                    throw new Error('NATIVE_INTEGRITY: Missing verified executable ' + executable);
+                chmodSync(join(output, executable), 0o755);
+            }
+            renameSync(output, destination);
+            return destination;
+        }
+        finally {
+            rmSync(stage, { recursive: true, force: true });
+        }
+    });
+}
+/** Extend the existing source lock with exact native target selections. */
+export async function prepareNativePackages(root, options = {}) {
+    const lockFile = join(root, 'aug.lock.json'), self = existsSync(join(root, 'aug-package.json')) ? readPackage(root) : undefined;
+    if (!existsSync(lockFile) && !self?.manifest.native)
+        return [];
+    return withPackageLockAsync(join(root, '.aug-install.lock'), async () => {
+        const initial = existsSync(lockFile) ? readFileSync(lockFile, 'utf8') : undefined;
+        const lock = initial ? JSON.parse(initial) : { format: 1, compiler: compilerVersion(), specifications: {}, roots: {}, packages: [], npm: {} };
+        if (lock.compiler !== compilerVersion())
+            throw new Error('NATIVE_LOCK: Source lock compiler differs; run aug install');
+        const target = options.target ?? nativeHostTarget(), key = target.triple + '/' + (target.os === 'macos' ? 'macos14' : target.libc);
+        const verified = projectPackages(root, lock.specifications);
+        if (verified.diagnostics.length)
+            throw new Error(verified.diagnostics.map(d => d.message).join('\n'));
+        const packages = [...verified.scopes.values()].filter(entry => entry.native).map(entry => ({ sourcePackage: entry.name + '@' + entry.version, sourceDigest: entry.digest,
+            sourceCommit: lock.git?.find(g => entry.path === 'packages/' + sourceAlias(g.request))?.commit,
+            contractSha256: entry.native.bindingsSha256, artifact: selectNativeArtifact(entry.native.artifacts, target) }));
+        if (self?.manifest.native) {
+            const digest = createHash('sha256');
+            for (const file of ['aug-package.json', 'native.abi.json', ...sourcePaths(self.sourceRoot).map(path => path.slice(root.length + 1))].sort())
+                digest.update(file + '\0').update(readFileSync(join(root, file))).update('\0');
+            packages.push({ sourcePackage: self.manifest.name + '@' + self.manifest.version, sourceDigest: digest.digest('hex'), sourceCommit: undefined, contractSha256: self.manifest.native.bindingsSha256, artifact: selectNativeArtifact(self.manifest.native.artifacts, target) });
+        }
+        if (!packages.length)
+            return [];
+        const native = lock.native ?? { format: 1, targets: {} };
+        if (native.format !== 1)
+            throw new Error('NATIVE_LOCK: Unsupported native lock schema');
+        const previous = native.targets[key];
+        if (options.frozen && (!previous || JSON.stringify(previous.packages) !== JSON.stringify(packages)))
+            throw new Error('NATIVE_LOCK: Frozen install has no matching native target lock. Run aug install online to record this target.');
+        const components = new Map();
+        for (const p of packages)
+            for (const c of p.artifact.components)
+                if (c.required) {
+                    const identity = c.id + '@' + c.version;
+                    if (components.has(c.compatibilityKey) && components.get(c.compatibilityKey) !== identity)
+                        throw new Error('NATIVE_CONFLICT: Incompatible native component ' + c.compatibilityKey);
+                    components.set(c.compatibilityKey, identity);
+                }
+        const inputs = [];
+        for (const p of packages) {
+            const directory = await ensureVerifiedArchive(p.artifact, { offline: options.offline });
+            const supplied = verifyArtifactFiles(directory, p.artifact);
+            for (const path of [...p.artifact.link.libraries, ...p.artifact.runtime.files, p.artifact.provenance, p.artifact.notices, ...(p.artifact.runtime.closureManifest ? [p.artifact.runtime.closureManifest] : [])])
+                if (!supplied[path])
+                    throw new Error('NATIVE_INTEGRITY: Artifact is missing declared file ' + path);
+            inputs.push({ directory, libraries: p.artifact.link.libraries, runtimeFiles: p.artifact.runtime.files, artifactSha256: p.artifact.sha256,
+                metadata: [...Object.keys(supplied).filter(path => !p.artifact.link.libraries.includes(path) && !p.artifact.runtime.files.includes(path)), p.artifact.fileManifest] });
+        }
+        if (packages.length && !options.frozen) {
+            native.targets[key] = { target, packages };
+            lock.native = native;
+            if ((existsSync(lockFile) ? readFileSync(lockFile, 'utf8') : undefined) !== initial)
+                throw new Error('NATIVE_LOCK: Source lock changed during native installation; retry aug install');
+            writeFileSync(lockFile + '.native.tmp', JSON.stringify(lock, null, 2) + '\n');
+            renameSync(lockFile + '.native.tmp', lockFile);
+        }
+        return inputs;
+    });
+}

package/src/native-bindings.js

@@ -0,0 +1,189 @@
+import { createHash } from 'node:crypto';
+import { existsSync, mkdtempSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, relative, resolve } from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { nativeTarget, validateNativeDescriptor } from "./native-contracts.js";
+const fail = (message) => { throw new Error('NATIVE_HEADER: ' + message); };
+const hash = (path) => createHash('sha256').update(readFileSync(path)).digest('hex');
+const scalar = (view) => ({ void: 'void', i64: 'int64_t', i32: 'int32_t', f64: 'double', bool: 'uint8_t' }[view.kind]);
+/** Check a maintainer-supplied ownership contract against real C declarations.
+ * Clang is invoked only by this explicit command, never by package installation.
+ * The result records physical checks, not proof of allocator or thread promises.
+ */
+export function bindNativeHeader(options) {
+    const header = resolve(options.header), contract = resolve(options.contract), output = resolve(options.output), flags = options.flags ?? [];
+    nativeTarget(options.target);
+    if (existsSync(output))
+        fail('Output already exists. Select a new directory; accepted bindings are never overwritten.');
+    const descriptor = validateNativeDescriptor(JSON.parse(readFileSync(contract, 'utf8'))), initialHeader = hash(header), initialContract = hash(contract);
+    const invoke = (args) => {
+        const result = spawnSync(options.clang, args, { encoding: 'utf8', timeout: 60000, maxBuffer: 32 * 1024 * 1024 });
+        if (result.status !== 0)
+            fail('Clang rejected the header or ABI probe. ' + (result.error?.message ?? result.stderr.split('\n').filter(line => !/^\.+ /.test(line)).join('\n').slice(0, 8000)));
+        return result;
+    };
+    const compiler = invoke(['--version']).stdout.trim();
+    if (!/clang version\s+(\d+)/.test(compiler))
+        fail('The selected executable is not Clang.');
+    // Force parsing and probing as C. C++ libraries expose a C adapter header.
+    const base = [...flags, '-target', options.target, '-std=c11', '-x', 'c'];
+    const includePaths = (stderr) => stderr.split('\n').flatMap(line => { const match = /^\.+ (.+)$/.exec(line); return match && existsSync(match[1]) ? [resolve(match[1])] : []; });
+    const parsed = invoke([...base, '-H', '-Xclang', '-ast-dump=json', '-fsyntax-only', header]);
+    const ast = JSON.parse(parsed.stdout);
+    const headerInputs = new Map([...new Set([header, ...includePaths(parsed.stderr)])].map(path => [path, hash(path)]));
+    const declarations = new Map(), records = new Map(), aliases = new Map();
+    const visit = (node) => {
+        if (node.kind === 'FunctionDecl' && node.name) {
+            const previous = declarations.get(node.name);
+            if (previous && previous.type?.qualType !== node.type?.qualType)
+                fail('Conflicting declarations for ' + node.name);
+            declarations.set(node.name, node);
+        }
+        if (node.kind === 'RecordDecl' && node.name) {
+            records.set(node.name, node);
+            if (node.id)
+                records.set(node.id, node);
+        }
+        if (node.kind === 'TypedefDecl' && node.name)
+            aliases.set(node.name, node);
+        for (const child of node.inner ?? [])
+            visit(child);
+    };
+    visit(ast);
+    const get = (symbol) => declarations.get(symbol) ?? fail('Missing C declaration for ' + symbol);
+    const params = (node) => (node.inner ?? []).filter(child => child.kind === 'ParmVarDecl').map(child => child.type.qualType);
+    const opaque = new Map();
+    for (const resource of descriptor.resources) {
+        const declaration = get(resource.release), arguments_ = params(declaration);
+        if (arguments_.length !== 1)
+            fail(resource.release + ' must release exactly one opaque pointer.');
+        const type = arguments_[0].replace(/\s+/g, ' ').trim();
+        if (!/^(?:struct )?[A-Za-z_]\w* \*$/.test(type) || type.startsWith('const '))
+            fail(resource.release + ' requires a mutable opaque pointer.');
+        const name = type.replace(/ \*$/, '').replace(/^struct /, '');
+        if (name !== 'void') {
+            let record = records.get(name);
+            const findRecord = (node) => { if (node.kind === 'RecordType' && node.decl)
+                record = records.get(node.decl.id); for (const child of node.inner ?? [])
+                findRecord(child); };
+            const alias = aliases.get(name);
+            if (alias)
+                findRecord(alias);
+            if (!record || record.completeDefinition)
+                fail('Resource ' + resource.name + ' requires an incomplete C record or void pointer; use an opaque adapter.');
+        }
+        opaque.set(resource.module + '.' + resource.name, type);
+    }
+    const checks = [], signatures = [];
+    const check = (symbol, result, arguments_) => {
+        get(symbol);
+        const index = checks.length;
+        checks.push(`typedef ${result} (*aug_check_${index})(${arguments_.join(', ') || 'void'});\n_Static_assert(__builtin_types_compatible_p(__typeof__(&${symbol}), aug_check_${index}), "ABI mismatch: ${symbol}");`);
+        signatures.push({ symbol, parameters: arguments_, result });
+    };
+    for (const resource of descriptor.resources)
+        check(resource.release, 'void', [opaque.get(resource.module + '.' + resource.name)]);
+    const input = (view, actual) => {
+        const primitive = scalar(view);
+        if (primitive)
+            return [primitive];
+        if (view.kind === 'resource') {
+            const type = opaque.get(view.resource), unqualified = actual.replace(/^const\s+/, '').replace(/\s+/g, ' ').trim();
+            if (unqualified !== type || view.ownership !== 'read' && /^const\s/.test(actual))
+                fail('Resource pointer or mutable loan differs for ' + view.resource);
+            return [actual];
+        }
+        if (view.kind === 'f64-list')
+            return ['const double *', 'uint64_t'];
+        if (view.kind === 'utf8-list')
+            return ['const void *const *', 'const uint64_t *', 'uint64_t'];
+        return ['const void *', 'uint64_t'];
+    };
+    for (const fn of descriptor.functions) {
+        const actual = params(get(fn.symbol)), arguments_ = [];
+        for (const view of fn.params)
+            arguments_.push(...input(view, actual[arguments_.length] ?? ''));
+        if (fn.status === 'i32') {
+            const result = scalar(fn.result);
+            if (fn.result.kind === 'resource')
+                arguments_.push(opaque.get(fn.result.resource) + '*');
+            else if (result && result !== 'void')
+                arguments_.push(result + ' *');
+            else if (!result)
+                arguments_.push(fn.result.kind === 'f64-list' ? 'double **' : 'void **', 'uint64_t *');
+            arguments_.push('aug_native_error_v1 *');
+        }
+        check(fn.symbol, fn.status === 'i32' ? 'int32_t' : scalar(fn.result), arguments_);
+        if (fn.result.release)
+            check(fn.result.release, 'void', [fn.result.kind === 'f64-list' ? 'double *' : 'void *', ...(fn.result.releaseLength ? ['uint64_t'] : [])]);
+    }
+    const stage = mkdtempSync(join(tmpdir(), 'aug-header-check-'));
+    try {
+        const probe = join(stage, 'probe.c');
+        const layout = descriptor.functions.some(fn => fn.status === 'i32') ? `
+_Static_assert(sizeof(aug_native_error_v1)==520, "error size");
+_Static_assert(_Alignof(aug_native_error_v1)==4, "error alignment");
+_Static_assert(offsetof(aug_native_error_v1,code)==0, "error code offset");
+_Static_assert(offsetof(aug_native_error_v1,message_length)==4, "error length offset");
+_Static_assert(offsetof(aug_native_error_v1,message)==8, "error buffer offset");
+_Static_assert(__builtin_types_compatible_p(__typeof__(((aug_native_error_v1*)0)->code),int32_t), "error code type");
+_Static_assert(__builtin_types_compatible_p(__typeof__(((aug_native_error_v1*)0)->message_length),uint32_t), "error length type");
+_Static_assert(__builtin_types_compatible_p(__typeof__(((aug_native_error_v1*)0)->message),unsigned char[512]), "error buffer type");` : '';
+        writeFileSync(probe, '#include <stdint.h>\n#include <stddef.h>\n#include ' + JSON.stringify(header) + '\n' + layout + '\n' + checks.join('\n') + '\n');
+        const verified = invoke([...base, '-Werror', '-H', '-fsyntax-only', probe]);
+        const included = [...new Set([header, ...includePaths(verified.stderr)])].sort();
+        const inputs = included.map(path => ({ path: relative(dirname(header), path).replaceAll('\\', '/'), sha256: hash(path) }));
+        for (const [path, digest] of headerInputs)
+            if (!existsSync(path) || hash(path) !== digest)
+                fail('An included header changed during validation; retry.');
+        if (hash(header) !== initialHeader || hash(contract) !== initialContract)
+            fail('Header or ownership contract changed during validation; retry.');
+        mkdirSync(dirname(output), { recursive: true });
+        const generated = mkdtempSync(join(dirname(output), '.aug-bind-'));
+        try {
+            writeFileSync(join(generated, 'native.abi.json'), JSON.stringify(descriptor, null, 2) + '\n');
+            writeFileSync(join(generated, 'header-check.json'), JSON.stringify({ format: 1, profile: descriptor.profile, target: options.target, compiler, flags, contractSha256: initialContract, inputs, signatures, errorLayout: layout ? { size: 520, alignment: 4, offsets: [0, 4, 8] } : undefined, ownership: 'author-declared', retention: 'author-declared', allocatorPairing: 'author-declared' }, null, 2) + '\n');
+            emitDeclarations(descriptor, generated);
+            renameSync(generated, output);
+        }
+        finally {
+            rmSync(generated, { recursive: true, force: true });
+        }
+    }
+    finally {
+        rmSync(stage, { recursive: true, force: true });
+    }
+}
+function emitDeclarations(descriptor, directory) {
+    const modules = new Map();
+    const module = (name) => { if (!modules.has(name))
+        modules.set(name, { imports: new Set(), lines: [] }); return modules.get(name); };
+    const type = (view, current) => {
+        if (view.kind === 'resource') {
+            const dot = view.resource.lastIndexOf('.'), source = view.resource.slice(0, dot), name = view.resource.slice(dot + 1);
+            if (source !== current)
+                module(current).imports.add(`import ${name} from ${source.replaceAll('/', '.')}`);
+            return name;
+        }
+        return ({ void: 'void', i64: 'int', i32: 'c_int', f64: 'float', bool: 'bool', utf8: 'string', bytes: 'Bytes', 'f64-list': 'List<float>', 'utf8-list': 'List<string>' })[view.kind];
+    };
+    for (const resource of descriptor.resources)
+        module(resource.module).lines.push('extern C resource ' + resource.name);
+    for (const fn of descriptor.functions) {
+        const parameters = fn.params.map(view => (view.ownership === 'consume' ? 'own ' : view.ownership === 'borrow' ? 'borrow ' : '') + type(view, fn.module) + ' ' + view.name);
+        let error = '';
+        if (fn.error) {
+            const dot = fn.error.lastIndexOf('.'), name = fn.error.slice(dot + 1), source = fn.error.slice(0, dot);
+            if (source !== fn.module)
+                module(fn.module).imports.add(`import ${name} from ${source.replaceAll('/', '.')}`);
+            error = ' unless ' + name;
+        }
+        module(fn.module).lines.push(`extern C ${fn.name}(${parameters.join(', ')}) returns ${fn.result.kind === 'resource' ? 'own ' : ''}${type(fn.result, fn.module)}${error}${fn.uses.length ? ' uses ' + fn.uses.join(' and ') : ''}${fn.changes.length ? ' changes ' + fn.changes.join(' and ') : ''}`);
+    }
+    for (const [name, value] of [...modules].sort(([a], [b]) => a.localeCompare(b, 'en'))) {
+        const file = join(directory, 'src', name + '.aug');
+        mkdirSync(dirname(file), { recursive: true });
+        writeFileSync(file, [...value.imports].sort().concat(value.lines).join('\n') + '\n');
+    }
+}