@halcyonlabs/app 0.47.0-halcyon.332 → 0.47.0-halcyon.334

package/dist/index.d.ts

@@ -191,10 +191,10 @@ declare const uiPreferenceDefinitions: {
     }>>;
     readonly "sidebar.threadGrouping.environment": UiPreferenceDefinition<z$1.ZodUnion<readonly [z$1.ZodLiteral<"auto">, z$1.ZodBoolean]>>;
     readonly "sidebar.chronologicalSort": UiPreferenceDefinition<z$1.ZodEnum<{
+        none: "none";
         updated: "updated";
         created: "created";
         alpha: "alpha";
-        none: "none";
     }>>;
     readonly "sidebar.sortDirection": UiPreferenceDefinition<z$1.ZodEnum<{
         default: "default";
@@ -6129,8 +6129,8 @@ declare const hostDaemonCommandRegistry: {
             skillFilePath: z$1.ZodString;
         }, z$1.core.$strict>], "kind">>;
         instructionMode: z$1.ZodEnum<{
-            append: "append";
             replace: "replace";
+            append: "append";
         }>;
         type: z$1.ZodLiteral<"thread.rewind.prepare">;
         leaseId: z$1.ZodString;
@@ -6278,8 +6278,8 @@ declare const hostDaemonCommandRegistry: {
             skillFilePath: z$1.ZodString;
         }, z$1.core.$strict>], "kind">>;
         instructionMode: z$1.ZodEnum<{
-            append: "append";
             replace: "replace";
+            append: "append";
         }>;
         type: z$1.ZodLiteral<"thread.start">;
         requestId: z$1.ZodString;
@@ -6742,8 +6742,8 @@ declare const hostDaemonCommandRegistry: {
                 envPassthrough: z$1.ZodArray<z$1.ZodString>;
             }, z$1.core.$strict>;
             instructionMode: z$1.ZodEnum<{
-                append: "append";
                 replace: "replace";
+                append: "append";
             }>;
             projectId: z$1.ZodString;
             providerId: z$1.ZodString;
@@ -6958,8 +6958,8 @@ declare const hostDaemonCommandRegistry: {
                 envPassthrough: z$1.ZodArray<z$1.ZodString>;
             }, z$1.core.$strict>;
             instructionMode: z$1.ZodEnum<{
-                append: "append";
                 replace: "replace";
+                append: "append";
             }>;
             projectId: z$1.ZodString;
             providerId: z$1.ZodString;
@@ -8131,8 +8131,8 @@ declare const hostDaemonCommandRegistry: {
         supported: z$1.ZodLiteral<true>;
         health: z$1.ZodObject<{
             status: z$1.ZodEnum<{
-                unknown: "unknown";
                 ready: "ready";
+                unknown: "unknown";
                 not_installed: "not_installed";
                 unauthenticated: "unauthenticated";
                 expired: "expired";
@@ -8183,9 +8183,9 @@ declare const hostDaemonCommandRegistry: {
         executablePath: z$1.ZodNullable<z$1.ZodString>;
         installed: z$1.ZodBoolean;
         installSource: z$1.ZodEnum<{
-            external: "external";
             notInstalled: "notInstalled";
             npmGlobal: "npmGlobal";
+            external: "external";
         }>;
         currentVersion: z$1.ZodNullable<z$1.ZodString>;
         latestVersion: z$1.ZodNullable<z$1.ZodString>;
@@ -8734,9 +8734,9 @@ declare const providerCliStatusResponseSchema: z$1.ZodRecord<z$1.ZodString, z$1.
     executablePath: z$1.ZodNullable<z$1.ZodString>;
     installed: z$1.ZodBoolean;
     installSource: z$1.ZodEnum<{
-        external: "external";
         notInstalled: "notInstalled";
         npmGlobal: "npmGlobal";
+        external: "external";
     }>;
     currentVersion: z$1.ZodNullable<z$1.ZodString>;
     latestVersion: z$1.ZodNullable<z$1.ZodString>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@halcyonlabs/app",
-  "version": "0.47.0-halcyon.332",
+  "version": "0.47.0-halcyon.334",
   "description": "bb app launcher for npx",
   "keywords": [
     "bb",

package/server/dist/builtin-plugins/orbit/dist/assets/orbit/bundle/scripts/orbit-job-supervisor

@@ -18,7 +18,9 @@ import subprocess
 RESULT_SPACE_FLOOR = 1_000_000
 MAIN_FREE_FLOOR = 15_000_000_000
 START_HEADROOM = 5_000_000_000
-EXPECTED_BUILD_BYTES = 5_500_000_000
+EXPECTED_JOB_BYTES = 1_600_000_000
+MIN_JOB_BYTES = 200_000_000
+JOB_HISTORY_LENGTH = 16
 # Many tiny files can exhaust inodes while bytes remain: the same rule applies
 # to free file entries on filesystems that have a fixed number of them.
 INODE_FLOOR = 500_000
@@ -389,6 +391,7 @@ class Workspaces:
                                        f"{gigabytes(settings['disk_free_bytes'] + retained)} GB there. Free space on it first.")
             # Persist ownership first. A crash before mkdir still leaves a reserved
             # attempt which restart must explicitly finish, never silently forget.
+            write_record(self.state, manifest['id'] + '.expected', self.expected_job_bytes())
             write_record(self.state, manifest['id'] + '.json', manifest, exclusive=True)
             if not workspace:
                 return manifest
@@ -404,28 +407,57 @@ class Workspaces:
             self.environment(manifest['id'])
             return manifest
 
+    def job_history(self):
+        raw = ledger_setting(self.state, 'job-footprints', 4096)
+        history = [] if raw is None else json.loads(raw)
+        if (not isinstance(history, list) or len(history) > JOB_HISTORY_LENGTH
+                or any(type(n) is not int or not 0 <= n <= 2**63 - 1 for n in history)):
+            raise ValueError('Invalid finished job footprints.')
+        return history
+
+    def expected_job_bytes(self, job_id=None):
+        if job_id is not None:
+            raw = ledger_setting(self.state, job_id + '.expected', 32)
+            if raw is not None:
+                value = json.loads(raw)
+                if type(value) is not int or not MIN_JOB_BYTES <= value <= 2**63 - 1:
+                    raise ValueError('Invalid job growth reservation.')
+                return value
+        history = self.job_history()
+        return min(EXPECTED_JOB_BYTES, max(MIN_JOB_BYTES, max(history) * 3 // 2)) if history else EXPECTED_JOB_BYTES
+
+    def remember_footprint(self, manifest):
+        if not has_workspace(manifest):
+            return
+        try:
+            usage = self.usage(manifest['id'])
+            if usage['complete']:
+                history = self.job_history()
+                write_record(self.state, 'job-footprints', (history + [usage['bytes']])[-JOB_HISTORY_LENGTH:])
+        except (OSError, ValueError):
+            return
+
     def start_space(self, records, settings, keep=None):
-        raw = ledger_setting(self.state, 'expected-build-bytes', 32)
-        previous = EXPECTED_BUILD_BYTES if raw is None else int(raw)
-        if not EXPECTED_BUILD_BYTES <= previous <= 2**63 - 1:
-            raise ValueError('Invalid measured build footprint.')
-        expected = max(previous, settings['disk_start_headroom_bytes'])
-        sizes = {}
+        expected = self.expected_job_bytes(keep)
+        outstanding = 0
+        kept_size = 0
+        deadline = time.monotonic() + 5
         for record in records:
             if not holds_slot(record) and record['id'] != keep:
                 continue
-            if has_workspace(record):
-                usage = self.usage(record['id'])
-                if not usage['complete']:
-                    raise CapacityFull('Expected build growth cannot be checked; waiting for a complete workspace measurement.')
-                sizes[record['id']] = usage['bytes']
-                expected = max(expected, usage['bytes'])
+            size = 0
+            if has_workspace(record) and time.monotonic() < deadline:
+                try:
+                    usage = self.usage(record['id'], deadline=deadline)
+                    if usage['complete']:
+                        size = usage['bytes']
+                except (OSError, ValueError):
+                    pass
+            if record['id'] == keep:
+                kept_size = size
             else:
-                sizes[record['id']] = 0
-        if expected > previous:
-            write_record(self.state, 'expected-build-bytes', expected)
-        outstanding = sum(max(0, expected - size) for job, size in sizes.items() if job != keep)
-        new_growth = max(0, expected - sizes.get(keep, 0))
+                outstanding += max(0, self.expected_job_bytes(record['id']) - size)
+        new_growth = max(settings['disk_start_headroom_bytes'], expected - kept_size)
         return settings['disk_free_bytes'] + outstanding + new_growth
 
     def require_admission(self, records):
@@ -492,7 +524,7 @@ class Workspaces:
 
 
 
-    def usage(self, job_id):
+    def usage(self, job_id, *, deadline=None):
         """Size a workspace to choose which job to stop when the disk is low.
 
         The walk is bounded to five seconds. A walk cut short by the time limit
@@ -505,23 +537,24 @@ class Workspaces:
         self.read(job_id)
         root = self.open_workspace(job_id)
         device = os.fstat(root).st_dev
-        stack = [(root, os.scandir(root))]
-        deadline = time.monotonic() + 5
+        stack = [(root, os.scandir(root), '')]
+        deadline = min(deadline, time.monotonic() + 5) if deadline is not None else time.monotonic() + 5
         try:
             return self._walk(stack, device, deadline)
         finally:
-            for fd, entries in reversed(stack):
+            for fd, entries, _ in reversed(stack):
                 entries.close()
                 os.close(fd)
 
     def _walk(self, stack, device, deadline):
         files = size = 0
+        complete = True
         seen = set()
         try:
             while stack:
                 if time.monotonic() > deadline:
                     return dict(files=files, bytes=size, complete=False)
-                current, entries = stack[-1]
+                current, entries, parent = stack[-1]
                 entry = next(entries, None)
                 if entry is None:
                     entries.close()
@@ -540,6 +573,15 @@ class Workspaces:
                     size += metadata.st_blocks * 512
                     seen.add(identity)
                 if stat.S_ISDIR(metadata.st_mode):
+                    if entry.name in MEASUREMENT_SKIPPED or (entry.name == 'build' and parent == 'results'):
+                        child = os.open(entry.name, DIR_FLAGS, dir_fd=current)
+                        try:
+                            with os.scandir(child) as skipped:
+                                if next(skipped, None) is not None:
+                                    complete = False
+                        finally:
+                            os.close(child)
+                        continue
                     try:
                         child = os.open(entry.name, DIR_FLAGS, dir_fd=current)
                     except OSError as error:
@@ -549,8 +591,8 @@ class Workspaces:
                     if os.fstat(child).st_dev != device:
                         os.close(child)
                         raise ValueError('A workspace cannot contain another filesystem.')
-                    stack.append((child, os.scandir(child)))
-            return dict(files=files, bytes=size, complete=True)
+                    stack.append((child, os.scandir(child), entry.name))
+            return dict(files=files, bytes=size, complete=complete)
         except OSError:
             return dict(files=files, bytes=size, complete=False)
 
@@ -784,6 +826,8 @@ class Workspaces:
             return self.save_transition(manifest, 'closed')
 
     def save_transition(self, manifest, state, reason=None):
+        if state == 'stopping' and reason == 'finished':
+            self.remember_footprint(manifest)
         updated = transition(manifest, state, reason)
         write_record(self.state, manifest['id'] + '.json', updated)
         return updated
@@ -1577,6 +1621,7 @@ def darwin_workspace_users(workspace):
 
 UNPUSHED_SKIPPED = frozenset(('.git', 'node_modules', 'target', '.orbit-build', 'cargo-home', 'cargo-target',
                               'npm-cache', 'pnpm-store', 'pnpm-home', 'cache'))
+MEASUREMENT_SKIPPED = frozenset(('node_modules', 'target', 'cargo-target', 'pnpm-store', 'npm-cache'))
 UNPUSHED_DEPTH = 6
 UNPUSHED_SECONDS = 30
 GIT_SECONDS = 20
@@ -1586,40 +1631,58 @@ GIT_SAFE = ('-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null', '-c'
 
 
 def git_checkouts(root, deadline, include_build=False):
-    def refuse(error):
-        raise error
-
     found = []
-    for directory, folders, files in os.walk(root, onerror=refuse):
+    pending = [Path(root)]
+    while pending:
         if time.monotonic() > deadline:
             raise ValueError('Searching the work folder for Git checkouts took too long.')
-        here = Path(directory)
-        checkout = '.git' in folders or '.git' in files or (include_build and {'HEAD', 'objects', 'config'}.issubset(set(folders) | set(files)))
-        if '.git' in folders and not (here / '.git').is_symlink():
-            with os.scandir(here / '.git') as entries:
-                checkout = next(entries, None) is not None
-        if checkout:
+        here = pending.pop()
+        folders = []
+        markers = set()
+        checkout = False
+        with os.scandir(here) as entries:
+            for entry in entries:
+                if time.monotonic() > deadline:
+                    raise ValueError('Searching the work folder for Git checkouts took too long.')
+                if entry.name in ('HEAD', 'objects', 'config'):
+                    markers.add(entry.name)
+                if entry.name == '.git':
+                    if entry.is_dir(follow_symlinks=False):
+                        with os.scandir(entry.path) as metadata:
+                            checkout = next(metadata, None) is not None
+                    else:
+                        checkout = True
+                if not entry.is_dir(follow_symlinks=False):
+                    continue
+                if entry.name in UNPUSHED_SKIPPED and not (include_build and entry.name == '.orbit-build'):
+                    continue
+                if here.name == 'results' and entry.name == 'build':
+                    continue
+                if include_build or len(here.relative_to(root).parts) < UNPUSHED_DEPTH:
+                    folders.append(entry.name)
+        if checkout or (include_build and {'HEAD', 'objects', 'config'}.issubset(markers)):
             found.append(here)
-        if not include_build and len(here.relative_to(root).parts) >= UNPUSHED_DEPTH:
-            folders[:] = []
-        else:
-            folders[:] = sorted(name for name in folders if name not in UNPUSHED_SKIPPED or (include_build and name == '.orbit-build'))
+        pending.extend(here / name for name in sorted(folders, reverse=True))
     return found
 
 
-def git_output(account, checkout, name, *args, allowed=(0,), stdin='', offline=False):
+def git_output(account, checkout, name, *args, allowed=(0,), stdin='', offline=False, deadline=None):
     import shutil
     import subprocess
     program = shutil.which('git', path=GIT_PATH)
     if program is None:
         raise ValueError('Git is not installed, so unpushed work cannot be checked.')
     environment = dict(PATH=GIT_PATH, HOME=account.pw_dir, LC_ALL='C', GIT_CONFIG_NOSYSTEM='1',
-                       GIT_CONFIG_GLOBAL='/dev/null', GIT_OPTIONAL_LOCKS='0', GIT_TERMINAL_PROMPT='0')
+                       GIT_CONFIG_GLOBAL='/dev/null', GIT_OPTIONAL_LOCKS='0', GIT_TERMINAL_PROMPT='0',
+                       GIT_CEILING_DIRECTORIES=str(Path(checkout).parent))
     if offline:
         environment.update(GIT_NO_LAZY_FETCH='1', GIT_ALLOW_PROTOCOL='', GIT_PAGER='cat')
+    timeout = GIT_SECONDS if deadline is None else min(GIT_SECONDS, deadline - time.monotonic())
+    if timeout <= 0:
+        raise ValueError('Checking the work folder for unpushed commits took too long.')
     try:
         result = subprocess.run([program, '-C', str(checkout), *GIT_SAFE, *args], cwd='/', env=environment, input=stdin,
-                                capture_output=True, text=True, errors='replace', timeout=GIT_SECONDS)
+                                capture_output=True, text=True, errors='replace', timeout=timeout)
     except subprocess.TimeoutExpired:
         raise ValueError(f'Git took too long to read {name}.')
     if result.returncode not in allowed:
@@ -1628,9 +1691,9 @@ def git_output(account, checkout, name, *args, allowed=(0,), stdin='', offline=F
     return result
 
 
-def unpushed_checkout(account, checkout, name):
+def unpushed_checkout(account, checkout, name, deadline=None):
     def git(*args, allowed=(0,)):
-        return git_output(account, checkout, name, *args, allowed=allowed)
+        return git_output(account, checkout, name, *args, allowed=allowed, deadline=deadline)
 
     def count(*revisions):
         return int(git('rev-list', '--count', *revisions).stdout.strip() or '0')
@@ -1666,7 +1729,7 @@ def search_unpushed(root, account, seconds, include_build=False):
         if time.monotonic() > deadline:
             raise ValueError('Checking the work folder for unpushed commits took too long.')
         name = str(checkout.relative_to(root))
-        work = unpushed_checkout(account, checkout, name)
+        work = unpushed_checkout(account, checkout, name, deadline)
         if work['branches'] or work['detached'] or work['uncommitted'] or work['untracked'] or work['stashes']:
             found.append(dict(path=name, **work))
     return dict(checkouts=found)

package/server/dist/builtin-plugins/orbit/dist/assets/orbit/revision

@@ -1 +1 @@
-52b039f31b9e0281a62dc3709e4e70e53d364cc1
+5844dd030759486b6f5044e4511c2b0082bde139

package/server/dist/start-server.js

@@ -117651,14 +117651,35 @@ async function runRemove(deps, environmentId, attempt, resumeOnly, signal) {
     }
   });
 }
+function canRetryWorkspaceRemoval(deps, row) {
+  if (row.ownerThreadId !== null || !row.providerOwnsPath || row.path === null || row.environmentProviderId === null)
+    return false;
+  const record2 = getEnvironmentProvider(row.environmentProviderId);
+  if (record2 === void 0 || record2.pluginId !== row.environmentProviderPluginId || !record2.provider.policy.internal_DO_NOT_USE_removeChecksWorkspaceUse)
+    return false;
+  return !["setup", "teardown"].some(
+    (kind) => deps.db.select({ id: environmentHookOperations.id }).from(environmentHookOperations).where(
+      and(
+        eq(
+          environmentHookOperations.id,
+          `environment:${row.id}:${row.environmentProviderInstanceKey}:${kind}`
+        ),
+        isNull(environmentHookOperations.finishedAt)
+      )
+    ).get() !== void 0
+  );
+}
 function writeEnvironmentTeardownFailure(deps, environmentId, message2) {
   const row = getEnvironment(deps.db, environmentId);
   if (row === null) return;
-  const exhausted = row.teardownAttempt >= MAX_REMOVE_ATTEMPTS;
+  const exhausted = row.teardownAttempt >= MAX_REMOVE_ATTEMPTS && !canRetryWorkspaceRemoval(deps, row);
   const written = deps.db.update(environments).set({
     teardownStatus: "failed",
     teardownMessage: exhausted ? `Needs recovery: ${message2} Inspect the workspace, then run bb environment cleanup ${environmentId} --recover.` : message2,
-    retireAt: exhausted ? null : Date.now() + REMOVE_RETRY_MS * 2 ** Math.max(0, row.teardownAttempt - 1),
+    retireAt: exhausted ? null : Date.now() + Math.min(
+      MAX_REMOVE_RETRY_MS,
+      REMOVE_RETRY_MS * 2 ** Math.min(6, Math.max(0, row.teardownAttempt - 1))
+    ),
     updatedAt: Date.now()
   }).where(
     and(
@@ -117737,7 +117758,7 @@ async function sweepProviderEnvironmentInSlot(deps, environmentId, signal) {
       });
     return;
   }
-  if (row.teardownAttempt >= MAX_REMOVE_ATTEMPTS) {
+  if (row.teardownAttempt >= MAX_REMOVE_ATTEMPTS && !canRetryWorkspaceRemoval(deps, row)) {
     if (row.retireAt !== null || !row.teardownMessage?.startsWith("Needs recovery:"))
       writeEnvironmentTeardownFailure(
         deps,
@@ -117746,6 +117767,14 @@ async function sweepProviderEnvironmentInSlot(deps, environmentId, signal) {
       );
     return;
   }
+  if (row.teardownStatus === "failed" && row.retireAt === null && canRetryWorkspaceRemoval(deps, row)) {
+    writeEnvironmentTeardownFailure(
+      deps,
+      environmentId,
+      row.teardownMessage ?? "Cleanup did not complete."
+    );
+    return;
+  }
   if (row.retireAt === null) {
     if (!cancelled && record2.provider.policy.retireGraceMs === null && row.status !== "destroyed")
       return;
@@ -117801,7 +117830,7 @@ function cleanupEnvironment(deps, environmentId) {
   if (row === null || row.environmentProviderId === null || !row.providerOwnsPath || environmentHasRetainedThreads(deps, row))
     return false;
   if (row.teardownStatus === "removed") return true;
-  if (row.teardownAttempt >= MAX_REMOVE_ATTEMPTS)
+  if (row.teardownAttempt >= MAX_REMOVE_ATTEMPTS && !canRetryWorkspaceRemoval(deps, row))
     throw new ApiError(
       409,
       "invalid_request",
@@ -118672,7 +118701,7 @@ async function forgetHostEnvironments(deps, hostId) {
 async function sweepProviderEnvironment(deps, environmentId) {
   await advanceEnvironmentProvisioning(deps, { environmentId, removal: true });
 }
-var resourceSchema2, createResultSchema2, removeResultSchema2, environmentOperations, REMOVE_RETRY_MS, MAX_REMOVE_ATTEMPTS, PROVIDER_OWNER_MISMATCH_MESSAGE, PROVIDER_LIFECYCLE_SWEEP_YIELD_INTERVAL, RETIRED_CREATE_CONTEXT_FIELDS;
+var resourceSchema2, createResultSchema2, removeResultSchema2, environmentOperations, REMOVE_RETRY_MS, MAX_REMOVE_ATTEMPTS, MAX_REMOVE_RETRY_MS, PROVIDER_OWNER_MISMATCH_MESSAGE, PROVIDER_LIFECYCLE_SWEEP_YIELD_INTERVAL, RETIRED_CREATE_CONTEXT_FIELDS;
 var init_environment_engine = __esm({
   "src/services/environments/environment-engine.ts"() {
     "use strict";
@@ -118732,6 +118761,7 @@ var init_environment_engine = __esm({
     environmentOperations = /* @__PURE__ */ new WeakMap();
     REMOVE_RETRY_MS = 6e4;
     MAX_REMOVE_ATTEMPTS = 5;
+    MAX_REMOVE_RETRY_MS = 36e5;
     PROVIDER_OWNER_MISMATCH_MESSAGE = "The environment provider belongs to a different plugin or has no recorded owner. Automatic removal is blocked.";
     PROVIDER_LIFECYCLE_SWEEP_YIELD_INTERVAL = 25;
     RETIRED_CREATE_CONTEXT_FIELDS = { rebuild: false, previous: null };