@halcyonlabs/app 0.47.0-halcyon.327 → 0.47.0-halcyon.328

package/dist/index.d.ts

@@ -83,8 +83,8 @@ declare const appSettingsSchema: z$1.ZodObject<{
     providerOrder: z$1.ZodArray<z$1.ZodString>;
     defaultProviderId: z$1.ZodNullable<z$1.ZodString>;
     providerCompletedTurnDisplay: z$1.ZodRecord<z$1.ZodString, z$1.ZodEnum<{
-        collapse: "collapse";
         flat: "flat";
+        collapse: "collapse";
     }>>;
     streamerMode: z$1.ZodBoolean;
     allowFastServiceTier: z$1.ZodBoolean;
@@ -105,8 +105,8 @@ declare const appSettingsUpdateSchema: z$1.ZodUnion<readonly [z$1.ZodObject<{
     providerOrder: z$1.ZodArray<z$1.ZodString>;
     defaultProviderId: z$1.ZodNullable<z$1.ZodString>;
     providerCompletedTurnDisplay: z$1.ZodRecord<z$1.ZodString, z$1.ZodEnum<{
-        collapse: "collapse";
         flat: "flat";
+        collapse: "collapse";
     }>>;
     streamerMode: z$1.ZodBoolean;
     managedBranchPrefix: z$1.ZodString;
@@ -143,8 +143,8 @@ declare const appSettingsUpdateSchema: z$1.ZodUnion<readonly [z$1.ZodObject<{
     providerOrder: z$1.ZodArray<z$1.ZodString>;
     defaultProviderId: z$1.ZodNullable<z$1.ZodString>;
     providerCompletedTurnDisplay: z$1.ZodRecord<z$1.ZodString, z$1.ZodEnum<{
-        collapse: "collapse";
         flat: "flat";
+        collapse: "collapse";
     }>>;
     streamerMode: z$1.ZodBoolean;
     machineServerUrl: z$1.ZodNullable<z$1.ZodString>;
@@ -316,8 +316,8 @@ declare const appKeybindingOverridesSchema: z$1.ZodArray<z$1.ZodObject<{
         shift: z$1.ZodBoolean;
     }, z$1.core.$strict>>;
     platform: z$1.ZodOptional<z$1.ZodEnum<{
-        windows: "windows";
         mac: "mac";
+        windows: "windows";
         linux: "linux";
     }>>;
 }, z$1.core.$strict>>;
@@ -3008,8 +3008,8 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
                 }>;
                 name: z$1.ZodString;
                 source: z$1.ZodEnum<{
-                    skill: "skill";
                     command: "command";
+                    skill: "skill";
                 }>;
                 origin: z$1.ZodEnum<{
                     user: "user";
@@ -3098,8 +3098,8 @@ declare const threadEventSchema: z$1.ZodUnion<readonly [z$1.ZodIntersection<z$1.
                 }>;
                 name: z$1.ZodString;
                 source: z$1.ZodEnum<{
-                    skill: "skill";
                     command: "command";
+                    skill: "skill";
                 }>;
                 origin: z$1.ZodEnum<{
                     user: "user";
@@ -4016,8 +4016,8 @@ declare const promptTextMentionSchema: z$1.ZodObject<{
         }>;
         name: z$1.ZodString;
         source: z$1.ZodEnum<{
-            skill: "skill";
             command: "command";
+            skill: "skill";
         }>;
         origin: z$1.ZodEnum<{
             user: "user";
@@ -4081,8 +4081,8 @@ declare const promptInputSchema: z$1.ZodDiscriminatedUnion<[z$1.ZodObject<{
             }>;
             name: z$1.ZodString;
             source: z$1.ZodEnum<{
-                skill: "skill";
                 command: "command";
+                skill: "skill";
             }>;
             origin: z$1.ZodEnum<{
                 user: "user";
@@ -4240,8 +4240,8 @@ declare const providerInfoSchema: z$1.ZodObject<{
         }, z$1.core.$strip>;
     }, z$1.core.$strip>], "kind">>;
     completedTurnDisplay: z$1.ZodEnum<{
-        collapse: "collapse";
         flat: "flat";
+        collapse: "collapse";
     }>;
     available: z$1.ZodBoolean;
     strings: z$1.ZodOptional<z$1.ZodObject<{
@@ -4329,8 +4329,8 @@ declare const threadTimelinePendingTodosSchema: z$1.ZodObject<{
 type ThreadTimelinePendingTodos = z$1.infer<typeof threadTimelinePendingTodosSchema>;
 
 declare const threadParentNotificationsSchema: z$1.ZodEnum<{
-    off: "off";
     auto: "auto";
+    off: "off";
     stuck: "stuck";
 }>;
 type ThreadParentNotifications = z$1.infer<typeof threadParentNotificationsSchema>;
@@ -4393,8 +4393,8 @@ declare const threadQueuedMessageSchema: z$1.ZodObject<{
                 }>;
                 name: z$1.ZodString;
                 source: z$1.ZodEnum<{
-                    skill: "skill";
                     command: "command";
+                    skill: "skill";
                 }>;
                 origin: z$1.ZodEnum<{
                     user: "user";
@@ -7860,13 +7860,13 @@ declare const hostDaemonCommandRegistry: {
         modifiedAtMs: z$1.ZodOptional<z$1.ZodNumber>;
         sha256: z$1.ZodString;
     }, z$1.core.$strip>, z$1.ZodObject<{
-        path: z$1.ZodString;
         contentEncoding: z$1.ZodEnum<{
             base64: "base64";
             utf8: "utf8";
         }>;
         sha256: z$1.ZodString;
         sizeBytes: z$1.ZodNumber;
+        path: z$1.ZodString;
         mimeType: z$1.ZodOptional<z$1.ZodString>;
         modifiedAtMs: z$1.ZodOptional<z$1.ZodNumber>;
         notModified: z$1.ZodLiteral<true>;
@@ -8131,8 +8131,8 @@ declare const hostDaemonCommandRegistry: {
         supported: z$1.ZodLiteral<true>;
         health: z$1.ZodObject<{
             status: z$1.ZodEnum<{
-                unknown: "unknown";
                 ready: "ready";
+                unknown: "unknown";
                 not_installed: "not_installed";
                 unauthenticated: "unauthenticated";
                 expired: "expired";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@halcyonlabs/app",
-  "version": "0.47.0-halcyon.327",
+  "version": "0.47.0-halcyon.328",
   "description": "bb app launcher for npx",
   "keywords": [
     "bb",

package/server/dist/builtin-plugins/orbit/dist/assets/orbit/bundle/scripts/orbit-job-supervisor

@@ -564,6 +564,49 @@ class Workspaces:
         os.close(self.open_workspace(job_id))
         return unpushed_work(self.workspace_root / (self.prefix + job_id), account)
 
+    def save_and_clean(self, job_id, account):
+        with self.locked():
+            manifest = self.read(job_id)
+            if not has_workspace(manifest) or manifest['state'] not in ('reserved', 'running', 'parked'):
+                raise ValueError('Only an open workspace can save and clean.')
+            path = self.workspace_root / (self.prefix + job_id)
+            users, unknown = workspace_users(path)
+            if users or unknown:
+                raise ValueError('Workspace writers must be confirmed stopped before saving and cleaning.')
+            manifest = self.declared_outputs(manifest)
+            root = self.open_workspace(job_id)
+            try:
+                checkouts = as_account(account, lambda: dict(paths=[str(p.relative_to(path)) for p in git_checkouts(path, time.monotonic() + UNPUSHED_SECONDS, include_build=True)]))['paths']
+                work = dict(checkouts=[dict(path=name, preserved=True) for name in checkouts])
+                caches = [tuple([name]) for name in JOB_CACHES] + declared_rebuildable(root, manifest['outputs'])
+                caches += [tuple(parts) for parts in as_account(account, lambda: dict(paths=recovery_caches(path, checkouts, account, manifest['outputs'])))['paths']]
+                safe_caches = []
+                for parts in caches:
+                    cache = '/'.join(parts)
+                    if any(checkout == cache or checkout.startswith(cache + '/') for checkout in checkouts):
+                        if parts == ('tmp',) or parts[-2:] == ('.orbit-build', 'tmp'):
+                            continue
+                        raise ValueError('A rebuildable folder contains a checkout; unpublished work is kept.')
+                    safe_caches.append(parts)
+                caches = safe_caches
+                name = 'results/storage-recovery-' + secrets.token_hex(16) + '.tar.gz'
+                outputs, _ = declared_output_names(manifest['outputs'] + [name], strict=True)
+                as_account(account, lambda: save_recovery(root, caches, account, name, outputs))
+                write_record(root, '.orbit-outputs.json', outputs)
+                declaration = os.open('.orbit-outputs.json', os.O_RDONLY | os.O_NOFOLLOW, dir_fd=root)
+                try:
+                    os.fchown(declaration, account.pw_uid, account.pw_gid)
+                finally:
+                    os.close(declaration)
+                manifest['outputs'] = outputs
+                write_record(self.state, manifest['id'] + '.json', manifest)
+                reclaim_workspace(root, outputs, caches)
+                if manifest['state'] != 'parked':
+                    self.save_transition(manifest, 'parked')
+                return dict(output=name, unpublished=work['checkouts'])
+            finally:
+                os.close(root)
+
     def measure(self, job_id, sized=True):
         """Free space and file entries for the stop rule; the folder's size only when low.
 
@@ -1064,13 +1107,50 @@ def declared_rebuildable(root, outputs):
     return paths[:REBUILDABLE_LIMIT]
 
 
-def reclaim_workspace(root, outputs=()):
+def cache_has_checkout(parent, name, device):
+    stack = []
+    deadline = time.monotonic() + 5
+    try:
+        stack.append((os.open(name, DIR_FLAGS, dir_fd=parent), None))
+        while stack:
+            current, entries = stack[-1]
+            if entries is None:
+                names = os.listdir(current)
+                if '.git' in names or {'HEAD', 'objects', 'config'}.issubset(names):
+                    return True
+                entries = iter(names)
+                stack[-1] = (current, entries)
+            entry = next(entries, None)
+            if entry is None:
+                os.close(current)
+                stack.pop()
+                continue
+            if time.monotonic() > deadline:
+                return True
+            info = os.stat(entry, dir_fd=current, follow_symlinks=False)
+            if info.st_dev != device:
+                return True
+            if stat.S_ISDIR(info.st_mode):
+                stack.append((os.open(entry, DIR_FLAGS, dir_fd=current), None))
+        return False
+    except (OSError, ValueError):
+        return True
+    finally:
+        for fd, _ in reversed(stack):
+            os.close(fd)
+
+
+def reclaim_workspace(root, outputs=(), paths=None):
     """Empty the caches the supervisor gives every job, then the folders the job declared rebuildable."""
     device = os.fstat(root).st_dev
     for name in JOB_CACHES:
-        if directory_at(root, name, device):
+        if paths is not None and (name,) not in paths:
+            continue
+        if directory_at(root, name, device) and not cache_has_checkout(root, name, device):
             clear_contents(root, name, device)
-    for parts in declared_rebuildable(root, outputs):
+    for parts in declared_rebuildable(root, outputs) if paths is None else paths:
+        if len(parts) == 1 and parts[0] in JOB_CACHES:
+            continue
         holder = os.dup(root)
         try:
             for part in parts[:-1]:
@@ -1080,12 +1160,140 @@ def reclaim_workspace(root, outputs=()):
                 os.close(holder)
                 holder = nested
             else:
-                if directory_at(holder, parts[-1], device):
+                if directory_at(holder, parts[-1], device) and not cache_has_checkout(holder, parts[-1], device):
                     remove_tree(holder, parts[-1], device)
         finally:
             os.close(holder)
 
 
+def recovery_caches(root, checkouts, account, outputs):
+    caches = []
+    for name in checkouts:
+        checkout = root / name
+        if not (checkout / '.git').exists():
+            continue
+        tracked = git_output(account, checkout, name, 'ls-files', '-z', offline=True).stdout.split('\0')
+        directories = {Path(file).parent for file in tracked if Path(file).name in ('package.json', 'Cargo.toml')}
+        candidates = [checkout / '.orbit-build' / folder for folder in (*JOB_CACHES, 'rustup', 'rustup-home', 'home/.rustup')]
+        candidates += [checkout / '.orbit-build/cargo-home' / folder for folder in ('registry', 'git')]
+        candidates += [checkout / directory / folder for directory in directories for folder in ('node_modules', 'target')]
+        for candidate in candidates:
+            relative = str(candidate.relative_to(root))
+            local = str(candidate.relative_to(checkout))
+            if (any(file == local or file.startswith(local + '/') for file in tracked if file)
+                    or any(output == relative or output.startswith(relative + '/') or relative.startswith(output + '/') for output in outputs)):
+                continue
+            caches.append(tuple(candidate.relative_to(root).parts))
+    return caches
+
+
+def workspace_metadata(root, excluded=()):
+    device = os.fstat(root).st_dev
+    def visit(directory, prefix):
+        for entry in os.listdir(directory):
+            parts = (*prefix, entry)
+            if any(parts[:len(path)] == path for path in excluded):
+                continue
+            info = os.stat(entry, dir_fd=directory, follow_symlinks=False)
+            if info.st_dev != device:
+                raise ValueError('Recovery source crosses a filesystem.')
+            yield parts, info
+            if stat.S_ISDIR(info.st_mode):
+                child = os.open(entry, DIR_FLAGS, dir_fd=directory)
+                try:
+                    yield from visit(child, parts)
+                finally:
+                    os.close(child)
+    yield from visit(root, ())
+
+
+def source_links(root, excluded):
+    shared = {(info.st_dev, info.st_ino): [file_identity(info), 0]
+              for _, info in workspace_metadata(root, excluded)
+              if stat.S_ISREG(info.st_mode) and info.st_nlink > 1}
+    if shared:
+        for _, info in workspace_metadata(root):
+            entry = shared.get((info.st_dev, info.st_ino))
+            if entry is not None:
+                entry[1] += 1
+        if any(identity[3] != count for identity, count in shared.values()):
+            raise ValueError('Source has a file name outside this workspace; originals are kept.')
+    return {key: identity for key, (identity, _) in shared.items()}
+
+
+def save_recovery(root, caches, account, name, outputs):
+    import tarfile
+    try:
+        os.mkdir('results', 0o700, dir_fd=root)
+    except FileExistsError:
+        pass
+    results = os.open('results', DIR_FLAGS, dir_fd=root)
+    os.fchown(results, account.pw_uid, account.pw_gid)
+    device = os.fstat(root).st_dev
+    declared = {tuple(output.split('/')) for output in outputs}
+    pending = '.storage-recovery-' + secrets.token_hex(16)
+    shared = source_links(root, [*caches, *declared])
+    descriptor = None
+    try:
+        if os.fstat(results).st_dev != device:
+            raise ValueError('Recovery output crosses a filesystem.')
+        descriptor = os.open(pending, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=results)
+        os.fchown(descriptor, account.pw_uid, account.pw_gid)
+        with os.fdopen(os.dup(descriptor), 'wb') as stream:
+            with tarfile.open(fileobj=stream, mode='w|gz') as archive:
+                def visit(directory, prefix):
+                    for entry in sorted(os.listdir(directory)):
+                        parts = (*prefix, entry)
+                        if (any(parts[:len(cache)] == cache for cache in caches)
+                                or parts == ('results', pending) or parts in declared):
+                            continue
+                        info = os.stat(entry, dir_fd=directory, follow_symlinks=False)
+                        if info.st_dev != device:
+                            raise ValueError('Recovery source crosses a filesystem.')
+                        member = tarfile.TarInfo('/'.join(parts))
+                        member.mode = info.st_mode & 0o777
+                        member.mtime = info.st_mtime
+                        if stat.S_ISDIR(info.st_mode):
+                            member.type = tarfile.DIRTYPE
+                            archive.addfile(member)
+                            child = os.open(entry, DIR_FLAGS, dir_fd=directory)
+                            try:
+                                visit(child, parts)
+                            finally:
+                                os.close(child)
+                        elif stat.S_ISLNK(info.st_mode):
+                            member.type = tarfile.SYMTYPE
+                            member.linkname = os.readlink(entry, dir_fd=directory)
+                            archive.addfile(member)
+                        elif stat.S_ISREG(info.st_mode):
+                            if info.st_nlink > 1 and shared.get((info.st_dev, info.st_ino)) != file_identity(info):
+                                raise ValueError('Source links changed; originals are kept.')
+                            fd = os.open(entry, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory)
+                            with os.fdopen(fd, 'rb') as source:
+                                if file_identity(os.fstat(source.fileno())) != file_identity(info):
+                                    raise ValueError('Recovery source changed; originals are kept.')
+                                member.size = info.st_size
+                                archive.addfile(member, source)
+                                if file_identity(os.fstat(source.fileno())) != file_identity(info):
+                                    raise ValueError('Recovery source changed; originals are kept.')
+                        else:
+                            raise ValueError('Recovery source is not a private file; originals are kept.')
+                visit(root, ())
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.link(pending, Path(name).name, src_dir_fd=results, dst_dir_fd=results, follow_symlinks=False)
+        os.unlink(pending, dir_fd=results)
+        os.fsync(results)
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        try:
+            os.unlink(pending, dir_fd=results)
+        except FileNotFoundError:
+            pass
+        os.close(results)
+
+
 def available(fd):
     disk = os.fstatvfs(fd)
     return disk.f_bavail * disk.f_frsize
@@ -1377,7 +1585,7 @@ GIT_SAFE = ('-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null', '-c'
             '-c', 'diff.external=', '-c', 'submodule.recurse=false')
 
 
-def git_checkouts(root, deadline):
+def git_checkouts(root, deadline, include_build=False):
     def refuse(error):
         raise error
 
@@ -1386,20 +1594,20 @@ def git_checkouts(root, deadline):
         if time.monotonic() > deadline:
             raise ValueError('Searching the work folder for Git checkouts took too long.')
         here = Path(directory)
-        checkout = '.git' in folders or '.git' in files
+        checkout = '.git' in folders or '.git' in files or (include_build and {'HEAD', 'objects', 'config'}.issubset(set(folders) | set(files)))
         if '.git' in folders and not (here / '.git').is_symlink():
             with os.scandir(here / '.git') as entries:
                 checkout = next(entries, None) is not None
         if checkout:
             found.append(here)
-        if len(here.relative_to(root).parts) >= UNPUSHED_DEPTH:
+        if not include_build and len(here.relative_to(root).parts) >= UNPUSHED_DEPTH:
             folders[:] = []
         else:
-            folders[:] = sorted(name for name in folders if name not in UNPUSHED_SKIPPED)
+            folders[:] = sorted(name for name in folders if name not in UNPUSHED_SKIPPED or (include_build and name == '.orbit-build'))
     return found
 
 
-def git_output(account, checkout, name, *args, allowed=(0,), stdin=''):
+def git_output(account, checkout, name, *args, allowed=(0,), stdin='', offline=False):
     import shutil
     import subprocess
     program = shutil.which('git', path=GIT_PATH)
@@ -1407,6 +1615,8 @@ def git_output(account, checkout, name, *args, allowed=(0,), stdin=''):
         raise ValueError('Git is not installed, so unpushed work cannot be checked.')
     environment = dict(PATH=GIT_PATH, HOME=account.pw_dir, LC_ALL='C', GIT_CONFIG_NOSYSTEM='1',
                        GIT_CONFIG_GLOBAL='/dev/null', GIT_OPTIONAL_LOCKS='0', GIT_TERMINAL_PROMPT='0')
+    if offline:
+        environment.update(GIT_NO_LAZY_FETCH='1', GIT_ALLOW_PROTOCOL='', GIT_PAGER='cat')
     try:
         result = subprocess.run([program, '-C', str(checkout), *GIT_SAFE, *args], cwd='/', env=environment, input=stdin,
                                 capture_output=True, text=True, errors='replace', timeout=GIT_SECONDS)
@@ -1449,10 +1659,10 @@ def unpushed_checkout(account, checkout, name):
     return dict(branches=branches, detached=detached, uncommitted=len(changed), untracked=len(untracked), stashes=stashes)
 
 
-def search_unpushed(root, account, seconds):
+def search_unpushed(root, account, seconds, include_build=False):
     deadline = time.monotonic() + seconds
     found = []
-    for checkout in git_checkouts(root, deadline):
+    for checkout in git_checkouts(root, deadline, include_build=include_build):
         if time.monotonic() > deadline:
             raise ValueError('Checking the work folder for unpushed commits took too long.')
         name = str(checkout.relative_to(root))
@@ -1641,6 +1851,10 @@ def managed_operation(store, account, request):
         os.close(fd)
     elif request['operation'] == 'unpushed':
         print(json.dumps(store.unpushed(request['id'], account)))
+    elif request['operation'] == 'save-clean':
+        if set(request) - {'operation', 'id', 'user', 'data'}:
+            raise ValueError('Saving and cleanup accept no command or build arguments.')
+        print(json.dumps(store.save_and_clean(request['id'], account)))
     elif request['operation'] == 'retire':
         store.retire(request['id'])
     elif request['operation'] == 'park':

package/server/dist/builtin-plugins/orbit/dist/assets/orbit/revision

@@ -1 +1 @@
-816da1404c5f550e8c8876501fb540d2c02669cf
+9b5d54c9c031f64c8ca64213bd2c5907735b0da9