@halcyonlabs/app 0.47.0-halcyon.113 → 0.47.0-halcyon.115

package/dist/index.d.ts

@@ -7456,8 +7456,8 @@ declare const hostDaemonCommandRegistry: {
             skillFilePath: z$1.ZodString;
         }, z$1.core.$strict>], "kind">>;
         instructionMode: z$1.ZodEnum<{
-            replace: "replace";
             append: "append";
+            replace: "replace";
         }>;
         type: z$1.ZodLiteral<"thread.rewind.prepare">;
         leaseId: z$1.ZodString;
@@ -7605,8 +7605,8 @@ declare const hostDaemonCommandRegistry: {
             skillFilePath: z$1.ZodString;
         }, z$1.core.$strict>], "kind">>;
         instructionMode: z$1.ZodEnum<{
-            replace: "replace";
             append: "append";
+            replace: "replace";
         }>;
         type: z$1.ZodLiteral<"thread.start">;
         requestId: z$1.ZodString;
@@ -8069,8 +8069,8 @@ declare const hostDaemonCommandRegistry: {
                 envPassthrough: z$1.ZodArray<z$1.ZodString>;
             }, z$1.core.$strict>;
             instructionMode: z$1.ZodEnum<{
-                replace: "replace";
                 append: "append";
+                replace: "replace";
             }>;
             projectId: z$1.ZodString;
             providerId: z$1.ZodString;
@@ -8285,8 +8285,8 @@ declare const hostDaemonCommandRegistry: {
                 envPassthrough: z$1.ZodArray<z$1.ZodString>;
             }, z$1.core.$strict>;
             instructionMode: z$1.ZodEnum<{
-                replace: "replace";
                 append: "append";
+                replace: "replace";
             }>;
             projectId: z$1.ZodString;
             providerId: z$1.ZodString;
@@ -9187,13 +9187,13 @@ declare const hostDaemonCommandRegistry: {
         modifiedAtMs: z$1.ZodOptional<z$1.ZodNumber>;
         sha256: z$1.ZodString;
     }, z$1.core.$strip>, z$1.ZodObject<{
-        path: z$1.ZodString;
         contentEncoding: z$1.ZodEnum<{
             base64: "base64";
             utf8: "utf8";
         }>;
         sha256: z$1.ZodString;
         sizeBytes: z$1.ZodNumber;
+        path: z$1.ZodString;
         mimeType: z$1.ZodOptional<z$1.ZodString>;
         modifiedAtMs: z$1.ZodOptional<z$1.ZodNumber>;
         notModified: z$1.ZodLiteral<true>;
@@ -9510,9 +9510,9 @@ declare const hostDaemonCommandRegistry: {
         executablePath: z$1.ZodNullable<z$1.ZodString>;
         installed: z$1.ZodBoolean;
         installSource: z$1.ZodEnum<{
+            external: "external";
             notInstalled: "notInstalled";
             npmGlobal: "npmGlobal";
-            external: "external";
         }>;
         currentVersion: z$1.ZodNullable<z$1.ZodString>;
         latestVersion: z$1.ZodNullable<z$1.ZodString>;
@@ -10061,9 +10061,9 @@ declare const providerCliStatusResponseSchema: z$1.ZodRecord<z$1.ZodString, z$1.
     executablePath: z$1.ZodNullable<z$1.ZodString>;
     installed: z$1.ZodBoolean;
     installSource: z$1.ZodEnum<{
+        external: "external";
         notInstalled: "notInstalled";
         npmGlobal: "npmGlobal";
-        external: "external";
     }>;
     currentVersion: z$1.ZodNullable<z$1.ZodString>;
     latestVersion: z$1.ZodNullable<z$1.ZodString>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@halcyonlabs/app",
-  "version": "0.47.0-halcyon.113",
+  "version": "0.47.0-halcyon.115",
   "description": "bb app launcher for npx",
   "keywords": [
     "bb",

package/server/dist/builtin-plugins/orbit/dist/assets/orbit/bundle/scripts/orbit-job-supervisor

@@ -10,6 +10,7 @@ import errno
 import hashlib
 import time
 import re
+import subprocess
 
 # A job has no size or file-count cap of its own. Work stops only when the disk
 # holding job folders falls below MAIN_FREE_FLOOR, which keeps room for the OS,
@@ -21,10 +22,9 @@ START_HEADROOM = 5_000_000_000
 # to free file entries on filesystems that have a fixed number of them.
 INODE_FLOOR = 500_000
 START_INODE_HEADROOM = 500_000
-MIN_CONCURRENT_JOBS = 2
-MAX_CONCURRENT_JOBS = 8
-JOB_MEMORY_BYTES = 6 * 2**30
-DESKTOP_MEMORY_BYTES = 6 * 2**30
+ADMISSION_DEFAULTS = dict(memory_free_bytes=2**30, disk_free_bytes=MAIN_FREE_FLOOR,
+                          disk_start_headroom_bytes=START_HEADROOM, free_inodes=INODE_FLOOR,
+                          start_inode_headroom=START_INODE_HEADROOM)
 CAPACITY_FULL_STATUS = 75
 LIMITS = {}
 # Records written before the free-space rule carry the old fixed caps. They load
@@ -219,59 +219,89 @@ def write_record(directory, name, value, exclusive=False):
         raise
 
 
-def machine_concurrency(memory, cores, override):
-    if override is not None:
-        if not re.fullmatch('[0-9]{1,9}', override):
-            raise ValueError('Job concurrency override must be a positive whole number.')
-        requested = int(override)
-        if requested < 1:
-            raise ValueError('Job concurrency override must be a positive whole number.')
-        slots = max(MIN_CONCURRENT_JOBS, min(MAX_CONCURRENT_JOBS, requested))
-        return slots, f'override {requested}; floor {MIN_CONCURRENT_JOBS}, ceiling {MAX_CONCURRENT_JOBS}'
-    memory_slots = max(0, memory - DESKTOP_MEMORY_BYTES) // JOB_MEMORY_BYTES
-    core_slots = cores // 3
-    slots = max(MIN_CONCURRENT_JOBS, min(MAX_CONCURRENT_JOBS, memory_slots, core_slots))
-    return slots, (f'{memory / 2**30:.1f} GiB RAM minus 6 GiB for desktop, antfly and main session; '
-                   f'6 GiB per job gives {memory_slots} slots; {cores} cores / 3 gives {core_slots}; '
-                   f'floor {MIN_CONCURRENT_JOBS}, ceiling {MAX_CONCURRENT_JOBS}')
-
-
-def machine_resources():
-    if sys.platform == 'darwin':
-        import subprocess
-        try:
-            memory = int(subprocess.check_output(['sysctl', '-n', 'hw.memsize'], timeout=5))
-        except subprocess.SubprocessError as error:
-            raise OSError('Machine memory reading failed.') from error
-    else:
-        memory = os.sysconf('SC_PHYS_PAGES') * os.sysconf('SC_PAGE_SIZE')
-    cores = os.cpu_count()
-    if memory <= 0 or cores is None or cores <= 0:
-        raise ValueError('Machine memory or core count is unavailable.')
-    return memory, cores
+def ledger_setting(state, name, maximum):
+    try:
+        fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK, dir_fd=state)
+    except FileNotFoundError:
+        return None
+    with os.fdopen(fd) as stream:
+        metadata = os.fstat(stream.fileno())
+        if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > maximum:
+            raise ValueError('Invalid admission settings file.')
+        value = stream.read(maximum + 1)
+        if len(value) > maximum:
+            raise ValueError('Invalid admission settings file.')
+        return value
+
+
+def admission_settings(state):
+    raw = ledger_setting(state, 'admission', 4096)
+    values = {} if raw is None else json.loads(raw)
+    if (not isinstance(values, dict) or set(values) - ADMISSION_DEFAULTS.keys()
+            or any(type(v) is not int or v <= 0 or v > 2**63 - 1 for v in values.values())):
+        raise ValueError('Admission settings must name positive whole-number pressure thresholds.')
+    if any(key != 'memory_free_bytes' and value < ADMISSION_DEFAULTS[key] for key, value in values.items()):
+        raise ValueError('Admission settings cannot lower the existing disk or file-entry floors.')
+    return dict(ADMISSION_DEFAULTS, **values)
 
 
 def concurrent_jobs(state):
-    try:
-        fd = os.open('concurrency', os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK, dir_fd=state)
-    except FileNotFoundError:
-        override = None
-    else:
-        with os.fdopen(fd) as stream:
-            metadata = os.fstat(stream.fileno())
-            if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > 32:
-                raise ValueError('Invalid job concurrency override file.')
-            override = stream.read(33).strip()
-    try:
-        memory, cores = machine_resources()
-        slots, reason = machine_concurrency(memory, cores, override)
-    except (OSError, ValueError) as error:
-        if override is not None:
-            slots, reason = machine_concurrency(0, 0, override)
-        else:
-            slots, reason = MIN_CONCURRENT_JOBS, f'machine reading unavailable: {error}; floor {MIN_CONCURRENT_JOBS}'
-    print(f'Job concurrency: {slots}; {reason}.', file=sys.stderr)
-    return slots
+    raw = ledger_setting(state, 'concurrency', 32)
+    if raw is None:
+        return None
+    value = raw.strip()
+    if not re.fullmatch('[0-9]{1,9}', value) or int(value) < 1:
+        raise ValueError('The temporary task ceiling must be a positive whole number.')
+    return int(value)
+
+
+def linux_available_memory(proc=Path('/proc'), cgroups=Path('/sys/fs/cgroup')):
+    fields = dict(line.split(':', 1) for line in (proc / 'meminfo').read_text().splitlines())
+    value = fields['MemAvailable'].split()
+    if len(value) != 2 or value[1] != 'kB':
+        raise ValueError('Invalid available memory reading.')
+    memory = int(value[0]) * 1024
+    membership = [line[3:] for line in (proc / 'self/cgroup').read_text().splitlines() if line.startswith('0::')]
+    if membership:
+        path = Path(membership[0])
+        if not path.is_absolute() or '..' in path.parts:
+            raise ValueError('Invalid memory group path.')
+        group = cgroups / str(path).lstrip('/')
+        while True:
+            try:
+                limit = (group / 'memory.max').read_text().strip()
+            except FileNotFoundError:
+                limit = 'max'
+            if limit != 'max':
+                maximum = int(limit)
+                current = int((group / 'memory.current').read_text())
+                if maximum < 0 or current < 0:
+                    raise ValueError('Invalid memory group reading.')
+                statistics = dict(line.split() for line in (group / 'memory.stat').read_text().splitlines())
+                reclaimable = max(0, int(statistics.get('inactive_file', '0'))
+                                  - int(statistics.get('file_dirty', '0'))
+                                  - int(statistics.get('file_writeback', '0')))
+                if reclaimable < 0 or reclaimable > current:
+                    raise ValueError('Invalid reclaimable memory reading.')
+                memory = min(memory, max(0, maximum - current + reclaimable))
+            if group == cgroups:
+                break
+            group = group.parent
+    if memory < 0:
+        raise ValueError('Invalid available memory reading.')
+    return memory
+
+
+def available_memory():
+    if sys.platform == 'linux':
+        return linux_available_memory()
+    if sys.platform == 'darwin':
+        reading = subprocess.check_output(['vm_stat'], timeout=5).decode()
+        page_size = int(re.search(r'page size of ([0-9]+) bytes', reading)[1])
+        pages = sum(int(re.search(r'^' + name + r':\s+([0-9]+)\.', reading, re.M)[1])
+                    for name in ('Pages free', 'Pages inactive', 'Pages speculative'))
+        return pages * page_size
+    raise OSError('Available memory is not supported on this system.')
 
 
 class Workspaces:
@@ -339,30 +369,30 @@ class Workspaces:
                 if workspace:
                     self.environment(existing['id'])
                 return existing
-            slots = [r for r in records if holds_slot(r)]
-            if len(slots) >= concurrent_jobs(self.state):
-                raise CapacityFull('Job capacity is reserved; wait for disposal.')
+            settings = self.require_admission(records)
             if free is None:
                 free = work_free(self)
+                inodes = work_inodes(self)
                 if workspace:
                     results_free = available(self.results)
-                    inodes = work_inodes(self)
             if not workspace:
-                if type(free) is not int or free < MAIN_FREE_FLOOR:
+                if inodes is not None and (type(inodes) is not int or inodes < settings['free_inodes']):
+                    raise CapacityFull(f'Station disk has {inodes} free file entries; waiting for free file entries.')
+                if type(free) is not int or free < settings['disk_free_bytes']:
                     raise CapacityFull(f'Station disk has {gigabytes(free)} GB free space; a start needs '
-                                       f'{gigabytes(MAIN_FREE_FLOOR)} GB. Archive handled work to free space.')
+                                       f"{gigabytes(settings['disk_free_bytes'])} GB. Archive handled work to free space.")
             else:
                 retained = sum(self.result_reservation(r) for r in records) + result_metadata_space(manifest)
-                if type(free) is not int or free < MAIN_FREE_FLOOR + START_HEADROOM:
+                if type(free) is not int or free < settings['disk_free_bytes'] + settings['disk_start_headroom_bytes']:
                     raise CapacityFull(f'Station disk has {gigabytes(free)} GB free space; a new job needs '
-                                       f'{gigabytes(MAIN_FREE_FLOOR + START_HEADROOM)} GB. Archive handled work to free space.')
-                if inodes is not None and (type(inodes) is not int or inodes < INODE_FLOOR + START_INODE_HEADROOM):
+                                       f"{gigabytes(settings['disk_free_bytes'] + settings['disk_start_headroom_bytes'])} GB. Archive handled work to free space.")
+                if inodes is not None and (type(inodes) is not int or inodes < settings['free_inodes'] + settings['start_inode_headroom']):
                     raise CapacityFull(f'Station disk has {inodes} free file entries; a new job needs '
-                                       f'{INODE_FLOOR + START_INODE_HEADROOM}. Archive handled work to free space.')
+                                       f"{settings['free_inodes'] + settings['start_inode_headroom']}. Archive handled work to free space.")
                 results_free = free if results_free is None else results_free
-                if type(results_free) is not int or results_free < MAIN_FREE_FLOOR + retained:
+                if type(results_free) is not int or results_free < settings['disk_free_bytes'] + retained:
                     raise CapacityFull(f'The saved-results disk has {gigabytes(results_free)} GB free space; a new job needs '
-                                       f'{gigabytes(MAIN_FREE_FLOOR + retained)} GB there. Free space on it first.')
+                                       f"{gigabytes(settings['disk_free_bytes'] + retained)} GB there. Free space on it first.")
             # Persist ownership first. A crash before mkdir still leaves a reserved
             # attempt which restart must explicitly finish, never silently forget.
             write_record(self.state, manifest['id'] + '.json', manifest, exclusive=True)
@@ -380,6 +410,22 @@ class Workspaces:
             self.environment(manifest['id'])
             return manifest
 
+    def require_admission(self, records):
+        settings = admission_settings(self.state)
+        ceiling = concurrent_jobs(self.state)
+        if ceiling is not None and sum(holds_slot(r) for r in records) >= ceiling:
+            raise CapacityFull('The temporary task ceiling is reached; wait for a task to finish or park.')
+        try:
+            memory = available_memory()
+            if type(memory) is not int or memory < 0:
+                raise ValueError('Invalid available memory reading.')
+        except (OSError, ValueError, KeyError, TypeError, IndexError, subprocess.SubprocessError) as error:
+            raise CapacityFull('Station available memory could not be checked; waiting for a memory reading.') from error
+        if memory < settings['memory_free_bytes']:
+            raise CapacityFull(f'Station has {gigabytes(memory)} GB available memory; a task needs '
+                               f"{gigabytes(settings['memory_free_bytes'])} GB. Waiting for memory to be freed.")
+        return settings
+
     def workspace_identity(self, job_id):
         if not self.prefix:
             return None
@@ -555,9 +601,10 @@ class Workspaces:
 
     def require_result_space(self, required):
         free = available(self.results)
-        if free < MAIN_FREE_FLOOR + required:
+        floor = admission_settings(self.state)['disk_free_bytes']
+        if free < floor + required:
             raise CapacityFull(f'The saved-results disk has {gigabytes(free)} GB free space; saving results needs '
-                               f'{gigabytes(MAIN_FREE_FLOOR + required)} GB there. Free space on it first.')
+                               f'{gigabytes(floor + required)} GB there. Free space on it first.')
 
     def read_output_declaration(self, root):
         fd = open_output(root, '.orbit-outputs.json')
@@ -613,8 +660,13 @@ class Workspaces:
                 return manifest
             if manifest['state'] != 'parked':
                 raise ValueError('Only a parked job can take a slot again.')
-            if len([r for r in self.records() if holds_slot(r)]) >= concurrent_jobs(self.state):
-                raise CapacityFull('Job capacity is reserved; wait for disposal.')
+            self.require_admission(self.records())
+            free, inodes, low = work_reading(self)
+            if low:
+                raise CapacityFull(f'Station disk has {gigabytes(free)} GB free space and {inodes} free file entries; '
+                                   'a waiting task needs the disk pressure to clear.')
+            if has_workspace(manifest):
+                self.require_result_space(sum(self.result_reservation(r) for r in self.records()))
             return self.save_transition(manifest, 'running')
 
     def retire(self, job_id):
@@ -899,7 +951,12 @@ def work_inodes(store):
 
 def work_reading(store):
     free, inodes = work_free(store), work_inodes(store)
-    return free, inodes, free < MAIN_FREE_FLOOR or (inodes is not None and inodes < INODE_FLOOR)
+    try:
+        settings = admission_settings(store.state)
+    except (OSError, ValueError):
+        settings = ADMISSION_DEFAULTS
+        print('Pressure settings could not be read; disk monitoring uses its default floors.', file=sys.stderr)
+    return free, inodes, free < settings['disk_free_bytes'] or (inodes is not None and inodes < settings['free_inodes'])
 
 
 def writable_directory(parent, name, expected=None, device=None):
@@ -1343,10 +1400,6 @@ def managed_operation(store, account, request):
         finally:
             os.close(root)
     elif request['operation'] == 'measure':
-        try:
-            concurrent_jobs(store.state)
-        except (OSError, ValueError) as error:
-            print(f'Job concurrency setting could not be read: {error}', file=sys.stderr)
         if request.get('size', True) not in (True, False):
             raise ValueError('Invalid measure request.')
         print(json.dumps(store.measure(request['id'], sized=request.get('size', True))))

package/server/dist/builtin-plugins/orbit/dist/assets/orbit/revision

@@ -1 +1 @@
-5f5bbcef81e9074d0c70249ce75f778b27538f4e
+24548d59c6e823a880b77272b3c83280929bec54