arcane-os 0.54.1 → 0.56.0

package/src/packager/core.mjs

@@ -13,6 +13,7 @@ import {
 import path from 'node:path';
 import {pathToFileURL} from 'node:url';
 import {appRelativeRoot,resolveAppRoot,resolvePackageOutputRoot} from '../app-layout.mjs';
+import {TARGET_IDS} from '../constants.mjs';
 import {readInstalledSdkLayout} from '../sdk-runtime-layout.mjs';
 import {withWorkspaceOperationLock} from '../workspace-operation-lock.mjs';
 import {materializeNativeModules} from './native-modules.mjs';
@@ -20,6 +21,7 @@ import {
     applyPwaEntryReferences,
     inspectImportMapHtml,
     readWorkspaceAssetVersion,
+    removePwaEntryReferences,
     resolveAssetReference,
     rewriteAssetReferences,
     versionAssetUrl
@@ -319,7 +321,7 @@ function normalizeOptionalRecord(value,label){
 export function validateAppConfig(value,appId,rootConfig,configPath=path.posix.join(appRelativeRoot(rootConfig,appId),APP_CONFIG_NAME)){
     assertOnlyKeys(value,new Set([
         'schemaVersion','id','displayName','version','entry','strategy','security',
-        'localAIModelPolicy','include','exclude','shared','adapter','pwa','outputDirectory','documents'
+        'localAIModelPolicy','include','exclude','shared','adapter','pwa','outputDirectory','documents','nativeResources'
     ]),`${appId}/${APP_CONFIG_NAME}`);
     if(value.schemaVersion!==1)fail(`${appId}/${APP_CONFIG_NAME}.schemaVersion must be 1.`);
     if(!is.string(value.id)||value.id!==appId||!APP_ID_PATTERN.test(value.id)){
@@ -334,6 +336,15 @@ export function validateAppConfig(value,appId,rootConfig,configPath=path.posix.j
     );
     const include=validatePathList(value.include,`${appId}/${APP_CONFIG_NAME}.include`,{required:true});
     const exclude=validatePathList(value.exclude??[],`${appId}/${APP_CONFIG_NAME}.exclude`);
+    let nativeResources;
+    if(value.nativeResources!==undefined){
+        const label=`${appId}/${APP_CONFIG_NAME}.nativeResources`;
+        assertOnlyKeys(value.nativeResources,new Set(['include','exclude']),label);
+        nativeResources={
+            include:validatePathList(value.nativeResources.include,`${label}.include`),
+            exclude:validatePathList(value.nativeResources.exclude??[],`${label}.exclude`)
+        };
+    }
     if(include.some(allowed=>sameOrDescendant(APP_CONFIG_NAME,allowed))){
         fail(`${appId}/${APP_CONFIG_NAME}.include must not expose the authored package configuration.`);
     }
@@ -381,6 +392,7 @@ export function validateAppConfig(value,appId,rootConfig,configPath=path.posix.j
         )}),
         include,
         exclude,
+        ...(nativeResources===undefined?{}:{nativeResources}),
         shared:[...value.shared],
         ...(adapter===undefined?{}:{adapter}),
         configPath
@@ -422,7 +434,8 @@ async function assertContainedRealPath(root,candidate,label){
     }
 }
 
-async function loadContext(requestedWorkspaceRoot,appId,{outputDirectory}={}){
+async function loadContext(requestedWorkspaceRoot,appId,{outputDirectory,target='browser'}={}){
+    if(!TARGET_IDS.includes(target))fail(`Unsupported package target: ${String(target)}.`);
     const workspaceRoot=await realDirectory(normalizeWorkspaceRoot(requestedWorkspaceRoot),'Workspace root');
     const rootConfigPath=path.join(workspaceRoot,ROOT_CONFIG_NAME);
     const rootConfig=validateRootConfig(await readJson(rootConfigPath,ROOT_CONFIG_NAME),rootConfigPath);
@@ -435,14 +448,19 @@ async function loadContext(requestedWorkspaceRoot,appId,{outputDirectory}={}){
     if(outputDirectory!==undefined){
         config.outputDirectory=normalizeRelativePath(outputDirectory,'package outputDirectory');
     }
+    const outputRoot=resolvePackageOutputRoot(workspaceRoot,rootConfig,config,{target,outputDirectory});
+    if(target!=='browser'&&config.nativeResources?.include.length>0){
+        config.outputDirectory=path.relative(workspaceRoot,outputRoot).split(path.sep).join('/');
+    }
     return {
         workspaceRoot,
         rootConfig,
         appsRoot,
         appRoot,
         appId,
+        target,
         config,
-        outputRoot:resolvePackageOutputRoot(workspaceRoot,rootConfig,config)
+        outputRoot
     };
 }
 
@@ -493,6 +511,11 @@ async function assertPackageOutputLocation(context){
     for(const selected of context.config.include){
         assertSelectedInput(context.appRoot,selected,context.config.exclude,context.appId);
     }
+    if(context.target!=='browser'&&context.config.nativeResources){
+        for(const selected of context.config.nativeResources.include){
+            assertSelectedInput(context.appRoot,selected,context.config.nativeResources.exclude,`${context.appId} native`);
+        }
+    }
     for(const sharedId of context.config.shared){
         for(const route of context.rootConfig.sharedPayloads[sharedId]){
             const sourceRoot=path.resolve(context.workspaceRoot,route.source);
@@ -525,6 +548,7 @@ async function collectSelectedPath({
     reject,
     records,
     destinations,
+    coalescedRecords,
     signal,
     label,
     allowRoot=false
@@ -554,6 +578,7 @@ async function collectSelectedPath({
                 reject,
                 records,
                 destinations,
+                coalescedRecords,
                 signal,
                 label
             });
@@ -566,9 +591,15 @@ async function collectSelectedPath({
         fail(`${label} overlaps the generated release manifest.`);
     }
     const key=pathKey(normalizedDestination);
-    if(destinations.has(key))fail(`Package destination collision: ${normalizedDestination}.`);
+    if(destinations.has(key)){
+        const existing=coalescedRecords?.get(key);
+        if(existing?.source===absolute&&existing.destination===normalizedDestination)return;
+        fail(`Package destination collision: ${normalizedDestination}.`);
+    }
     destinations.add(key);
-    records.push({source:absolute,destination:normalizedDestination});
+    const record={source:absolute,destination:normalizedDestination};
+    records.push(record);
+    coalescedRecords?.set(key,record);
 }
 
 async function collectPackageRecords(context,{signal}={}){
@@ -612,6 +643,29 @@ async function collectPackageRecords(context,{signal}={}){
             }
         }
     }
+    if(context.target!=='browser'&&context.config.nativeResources){
+        const nativeRecords=[];
+        const coalescedRecords=new Map(records.map(record=>[pathKey(record.destination),record]));
+        for(const selected of context.config.nativeResources.include){
+            await collectSelectedPath({
+                sourceRoot:context.appRoot,
+                selected,
+                destination:appPackagePath(context,selected),
+                excludes:context.config.nativeResources.exclude,
+                reject:isAppSourceForbidden,
+                records:nativeRecords,
+                destinations,
+                coalescedRecords,
+                signal,
+                label:`${context.appId} nativeResources`
+            });
+        }
+        for(const record of nativeRecords){
+            record.appRelativePath=path.relative(context.appRoot,record.source).split(path.sep).join('/');
+            record.nativeOnly=true;
+            records.push(record);
+        }
+    }
     records.sort((left,right)=>compareText(left.destination,right.destination));
     if(!records.some(record=>pathKey(record.destination)===pathKey(appPackagePath(context, context.config.entry)))){
         fail(`Package entry is missing from the selected files: ${context.config.entry}.`);
@@ -670,7 +724,7 @@ async function optionalDescriptor(context){
 
 async function inspectContext(context,{signal,records}={}){
     const selectedRecords=records??await collectPackageRecords(context,{signal});
-    const documents=await browserDocuments(selectedRecords,context.config.entry,context.config.documents);
+    const documents=await browserDocuments(selectedRecords.filter(record=>!record.nativeOnly),context.config.entry,context.config.documents);
     return {
         appId:context.appId,
         displayName:context.config.displayName,
@@ -682,6 +736,7 @@ async function inspectContext(context,{signal,records}={}){
         ...(context.config.pwa===undefined?{}:{pwa:copyJson(context.config.pwa)}),
         include:[...context.config.include],
         exclude:[...context.config.exclude],
+        ...(context.config.nativeResources===undefined?{}:{nativeResources:copyJson(context.config.nativeResources)}),
         shared:[...context.config.shared],
         ...(context.config.security===undefined?{}:{security:copyJson(context.config.security)}),
         ...(context.config.localAIModelPolicy===undefined?{}:{
@@ -723,9 +778,9 @@ export async function discoverApps({workspaceRoot:requestedWorkspaceRoot}={}){
     return apps;
 }
 
-export async function inspectApp({workspaceRoot,appId,signal}={}){
+export async function inspectApp({workspaceRoot,appId,signal,target,outputDirectory}={}){
     throwIfAborted(signal);
-    const context=await loadContext(workspaceRoot,appId);
+    const context=await loadContext(workspaceRoot,appId,{target,outputDirectory});
     return inspectContext(context,{signal});
 }
 
@@ -845,7 +900,8 @@ async function replaceDirectory(stagingRoot,outputRoot){
 }
 
 async function packageWithContext(context,options={}){
-    const {signal,onEvent,browserPwa=true,moduleFormat='import-map'}=options;
+    const {signal,onEvent,moduleFormat='import-map',browserPwa:requestedBrowserPwa=true}=options;
+    const browserPwa=context.target==='browser'&&requestedBrowserPwa;
     if(!['import-map','native'].includes(moduleFormat)){
         throw new TypeError('packageApp moduleFormat must be "import-map" or "native".');
     }
@@ -853,7 +909,9 @@ async function packageWithContext(context,options={}){
     const appPath=appRelativeRoot(context.rootConfig,context.appId);
     const entryPath=appPackagePath(context,context.config.entry);
     const records=await collectPackageRecords(context,{signal});
-    const rootDocument = records.some(
+    const browserRecords=records.filter(record=>!record.nativeOnly);
+    const nativeRecords=records.filter(record=>record.nativeOnly);
+    const rootDocument = browserRecords.some(
         function selectedRootIndex(record) {
             return record.destination === 'index.html';
         }
@@ -900,7 +958,7 @@ async function packageWithContext(context,options={}){
     let promoted=false;
     try{
         async function copyBase() {
-            await copyRecords(records,stagingRoot,{signal,onEvent});
+            await copyRecords(browserRecords,stagingRoot,{signal,onEvent});
             if (!rootDocument) {
                 await writePackageLauncher(context, stagingRoot);
             }
@@ -912,6 +970,8 @@ async function packageWithContext(context,options={}){
                 workspaceRoot:context.workspaceRoot,
                 appRoot:context.appRoot,
                 outputRoot:stagingRoot,
+                target:context.target,
+                browserPwa,
                 copyBase,
                 signal,
                 onEvent
@@ -920,12 +980,57 @@ async function packageWithContext(context,options={}){
             await copyBase();
         }
         let files=await listOutputFiles(stagingRoot,{signal});
+        const entryUrl=new URL(packageResourceUrl(entryPath),'http://arcane.invalid/');
+        const selectedFiles=new Set(files);
+        const explicitDocumentPaths=context.config.documents===undefined?null:new Set(
+            inspected.browserDocuments.map(function selectedDocumentPath(document){return document.packagePath;})
+        );
+        const pwaDocumentPaths=new Set(explicitDocumentPaths??[entryPath]);
+        for(const selected of explicitDocumentPaths===null?context.config.include:[]){
+            const selectedPath=appPackagePath(context,selected);
+            if(/\.html?$/iu.test(selected)&&selectedFiles.has(selectedPath))pwaDocumentPaths.add(selectedPath);
+        }
+        for(const document of inspected.browserDocuments){
+            const appDocument=context.config.include.some(function includesAppDocument(selected){
+                return sameOrDescendant(document.path,selected);
+            });
+            if(appDocument&&(appPath===''||document.managedMaps.length>0)&&selectedFiles.has(document.packagePath)){
+                pwaDocumentPaths.add(document.packagePath);
+            }
+        }
+        if (!pwaEnabled) {
+            // Remove source-generated registration before native processing can follow it.
+            const removals = await Promise.allSettled(
+                [...pwaDocumentPaths].map(
+                    async function removeStagedPwaReferences(documentPath) {
+                        throwIfAborted(signal);
+                        const filePath = path.join(stagingRoot, ...documentPath.split('/'));
+                        const source = await readFile(filePath, 'utf8');
+                        const content = removePwaEntryReferences(
+                            source,
+                            {
+                                documentUrl: new URL(packageResourceUrl(documentPath), entryUrl.origin),
+                                manifestUrl: new URL(`/${PWA_MANIFEST_NAME}`, entryUrl.origin)
+                            }
+                        );
+                        if (content !== source) await writeFile(filePath, content, 'utf8');
+                    }
+                )
+            );
+            // Settle every stage write before a failure can trigger stage cleanup.
+            const errors = [];
+            for (const removal of removals) {
+                if (removal.status === 'rejected') errors.push(removal.reason);
+            }
+            if (errors.length === 1) throw errors[0];
+            if (errors.length > 1) throw new AggregateError(errors, 'Selected PWA reference removal failed.');
+        }
         if(moduleFormat==='native'){
             files=await materializeNativeModules({
                 stagingRoot,
                 files,
                 documents:inspected.browserDocuments.map(function documentPath(document){return document.packagePath;}),
-                sharedFiles:records.filter(function sharedRecord(record){return record.appRelativePath===undefined;})
+                sharedFiles:browserRecords.filter(function sharedRecord(record){return record.appRelativePath===undefined;})
                     .map(function sharedPath(record){return record.destination;}),
                 signal,
                 onEvent
@@ -937,27 +1042,10 @@ async function packageWithContext(context,options={}){
         const inventory=new Set(files);
         const offlineInventory=pwaEnabled?new Set(selectPwaFiles(files,context.config.pwa,appPath)):null;
         const offlineReferences=new Set();
-        const entryUrl=new URL(packageResourceUrl(entryPath),'http://arcane.invalid/');
         const pwaDocumentSources=new Map();
-        const explicitDocumentPaths=context.config.documents===undefined?null:new Set(
-            inspected.browserDocuments.map(function selectedDocumentPath(document){return document.packagePath;})
-        );
         if(pwaEnabled){
-            const documentPaths=new Set(explicitDocumentPaths??[entryPath]);
-            for(const selected of explicitDocumentPaths===null?context.config.include:[]){
-                const selectedPath=appPackagePath(context,selected);
-                if(/\.html?$/iu.test(selected)&&inventory.has(selectedPath))documentPaths.add(selectedPath);
-            }
-            for(const document of inspected.browserDocuments){
-                const appDocument=context.config.include.some(function includesAppDocument(selected){
-                    return sameOrDescendant(document.path,selected);
-                });
-                if(appDocument&&(appPath===''||document.managedMaps.length>0)&&inventory.has(document.packagePath)){
-                    documentPaths.add(document.packagePath);
-                }
-            }
             await Promise.all(
-                [...documentPaths].map(
+                [...pwaDocumentPaths].map(
                     async function readPwaDocument(documentPath) {
                         const source = await readFile(
                             path.join(stagingRoot, ...documentPath.split('/')),
@@ -978,8 +1066,8 @@ async function packageWithContext(context,options={}){
         const documentUrl=entryDocument?.bases[0]?.href
             ?new URL(entryDocument.bases[0].href,entryUrl):entryUrl;
         const pending=[{file:entryPath,documentUrl}];
-        const sharedFiles=new Set(records.filter(record=>record.appRelativePath===undefined).map(record=>record.destination));
-        const applicationFiles=new Set(records.filter(function applicationRecord(record){
+        const sharedFiles=new Set(browserRecords.filter(record=>record.appRelativePath===undefined).map(record=>record.destination));
+        const applicationFiles=new Set(browserRecords.filter(function applicationRecord(record){
             return record.appRelativePath!==undefined;
         }).map(function applicationDestination(record){return record.destination;}));
         for(const file of files){
@@ -1103,6 +1191,17 @@ async function packageWithContext(context,options={}){
             }
             files.sort(compareText);
         }
+        // Native resources bypass browser rewriting, module projection and PWA generation.
+        // The app's browser selection remains authoritative wherever both select a file.
+        const finalDestinations=new Set(files.map(pathKey));
+        for(const record of nativeRecords){
+            if(finalDestinations.has(pathKey(record.destination))){
+                fail(`Native resource overlaps generated package content: ${record.destination}.`);
+            }
+        }
+        await copyRecords(nativeRecords,stagingRoot,{signal,onEvent});
+        files.push(...nativeRecords.map(record=>record.destination));
+        files.sort(compareText);
         const manifest=releaseManifest(context,files,pwaArtifacts,rootDocument);
         await writeFile(
             path.join(stagingRoot,RELEASE_MANIFEST_NAME),

package/src/process.mjs

@@ -33,8 +33,8 @@ function platformArguments(command,args,platform=process.platform){
     return [cliPath,...args];
 }
 
-function appendOutput(current,chunk){
-    return current+chunk.toString('utf8');
+function appendOutput(current,text){
+    return current+text;
 }
 
 async function emitLines(events,type,text){
@@ -45,9 +45,9 @@ async function emitLines(events,type,text){
     }
 }
 
-function deliverChunk(stream,events,type,chunk){
+function deliverChunk(stream,events,type,text){
     stream.pause();
-    return emitLines(events,type,chunk.toString('utf8')).catch(()=>{
+    return emitLines(events,type,text).catch(()=>{
         // The queue owns and propagates the first callback failure.
     }).finally(()=>{
         if(!stream.destroyed){
@@ -263,6 +263,10 @@ export async function runProcess(command,args=[],{
             return;
         }
 
+        // Each readable retains split UTF-8 sequences and flushes at EOF.
+        // Capture and events consume the same decoded text.
+        child.stdout.setEncoding('utf8');
+        child.stderr.setEncoding('utf8');
         child.stdout.on('data',chunk=>{
             stdout=appendOutput(stdout,chunk);
             ownDelivery(child.stdout,'process.stdout',chunk);

package/src/targets/index.mjs

@@ -21,7 +21,7 @@ const DEFINITIONS=[
         id:'portable',displayName:'Portable native application',status:'pairing-required',
         platforms:['windows','linux'],architectures:['x64','arm64'],formats:['portable'],
         signingModes:['unsigned-local-test'],
-        reason:'Portable output is available when paired to an Arcane OS native provider.'
+        reason:'Portable app payloads use the SDK package provider by default; explicit provider pairing remains available.'
     },
     {
         id:'windows-x64',displayName:'Windows x64 executable',status:'pairing-required',
@@ -226,6 +226,7 @@ export function createNativeTargetAdapter({targetId,nativeBuilder}={}){
         appReleaseRoot,
         release,
         appDescriptor,
+        selectedSdk,
         dependencyReleases,
         minimumCoreVersion,
         protectedRoots,
@@ -242,6 +243,7 @@ export function createNativeTargetAdapter({targetId,nativeBuilder}={}){
             appReleaseRoot,
             release,
             appDescriptor,
+            selectedSdk,
             dependencyReleases,
             minimumCoreVersion,
             protectedRoots,

package/src/toolchain.mjs

@@ -836,8 +836,11 @@ function sameCanonicalPath(left,right){
     return normalize(left)===normalize(right);
 }
 
-export function assertIntegratedNativeToolchain({workspaceMode,workspaceRoot,toolchainRoot,target}={}){
+export function assertIntegratedNativeToolchain({
+    workspaceMode,workspaceRoot,toolchainRoot,target,providerSource='arcane-checkout'
+}={}){
     if(workspaceMode!=='integrated')return;
+    if(providerSource==='sdk-package'&&target==='portable')return;
     if(!is.string(toolchainRoot)||!sameCanonicalPath(workspaceRoot,toolchainRoot)){
         throw new ArcaneError(
             ERROR_CODES.policyDenied,
@@ -992,12 +995,15 @@ function pathsOverlap(left,right){
 
 async function packageNativeRelease(prepared,app,{
     workspaceOperationLease,
+    selectedSdk,
+    target,
     signal,
     onEvent
 }={}){
     const release=await packageApp({
         workspaceRoot:prepared.workspaceRoot,
         appId:app.appId,
+        target,
         browserPwa:false,
         workspaceOperationLease,
         signal,
@@ -1010,6 +1016,7 @@ async function packageNativeRelease(prepared,app,{
         release,
         nativeInput:{
             appDescriptor:app.descriptor,
+            ...(selectedSdk===undefined?{}:{selectedSdk}),
             appReleaseRoot:release.outputRoot??path.resolve(prepared.workspaceRoot,release.output),
             release:{manifest:release.manifest,files:[...release.files]}
         }
@@ -1027,7 +1034,8 @@ async function executePairedNativeBuild(options,adapter,{
         target,
         workspaceMode:initialPrepared.workspaceMode,
         workspaceRoot:initialPrepared.workspaceRoot,
-        toolchainRoot:options.toolchainRoot
+        toolchainRoot:options.toolchainRoot,
+        providerSource:options.providerSource
     });
     if(options.dryRun){
         throw new ArcaneError(
@@ -1051,10 +1059,15 @@ async function executePairedNativeBuild(options,adapter,{
     }
     const dependencyApps=await nativeDependencyClosure(initialPrepared,{target,signal,onEvent});
     const prepared=initialPrepared;
+    const installedSdk=await readInstalledSdkLayout(prepared.workspaceRoot,prepared.validation.config);
+    const selectedSdk=installedSdk===null?undefined:{
+        packageRoot:installedSdk.packageRoot,
+        packageSource:installedSdk.packageSource
+    };
     const packageOutputs=[];
     for(const app of [prepared.validation.app,...dependencyApps]){
         const releaseRoot=resolvePackageOutputRoot(
-            prepared.workspaceRoot,prepared.validation.config,app.manifest
+            prepared.workspaceRoot,prepared.validation.config,app.manifest,{target}
         );
         if(pathsOverlap(releaseRoot,outputRoot)
             ||packageOutputs.some(existing=>pathsOverlap(existing,releaseRoot))){
@@ -1073,13 +1086,13 @@ async function executePairedNativeBuild(options,adapter,{
         descriptorSource:prepared.validation.app.descriptorSource,
         descriptorPath:prepared.validation.app.descriptorPath,
         validation:prepared.validation
-    },{workspaceOperationLease,signal,onEvent});
+    },{workspaceOperationLease,selectedSdk,target,signal,onEvent});
     const dependencyReleases=[];
     for(const dependency of dependencyApps){
         dependencyReleases.push(await packageNativeRelease(
             prepared,
             dependency,
-            {workspaceOperationLease,signal,onEvent}
+            {workspaceOperationLease,selectedSdk,target,signal,onEvent}
         ));
     }
     const toolchain=await adapter.prepare({
@@ -1094,7 +1107,11 @@ async function executePairedNativeBuild(options,adapter,{
         toolchainRoot:options.toolchainRoot,
         toolchain,
         ...selectedRelease.nativeInput,
-        dependencyReleases:dependencyReleases.map(item=>item.nativeInput),
+        dependencyReleases:dependencyReleases.map(item=>({
+            appId:item.appId,
+            releaseRoot:item.nativeInput.appReleaseRoot,
+            release:item.nativeInput.release
+        })),
         minimumCoreVersion:ARCANE_MACHINE_BUNDLE_VERSION,
         protectedRoots,
         outputRoot,