@gotcos/glasses-server 6.77.0 → 6.78.1

package/server/routes/meeting.ts

@@ -181,6 +181,7 @@ import {
   type MeetingFinalizationJob,
 } from '../lib/meeting-finalization-jobs.js'
 import { enrichStandaloneMeeting } from '../lib/meeting-summary-persistence.js'
+import { bootResumeGate, type BootResumeGate } from '../lib/boot-resume.js'
 import { recordFinalPassSkip, runNemotronFinalPass, type FinalPassOptions, type FinalPassOutcome } from '../lib/nemotron-final.js'
 import { runTranscriptCleanup, type CleanupOutcome } from '../lib/transcript-cleanup.js'
 import { sweepTranscriptCleanupData } from '../lib/transcript-cleanup-screen.js'
@@ -312,6 +313,7 @@ function scheduleFinalizationJob(job: MeetingFinalizationJob, runtime: Finalizat
     let current = runtime.finalizationJobs.get(job.sessionId) ?? job
     if (canonicalFinalizationIsComplete(current)) {
       runtime.finalizationJobs.remove(current.sessionId)
+      bootResumeGate().forget('meeting_finalization', current.sessionId)
       return
     }
     if (current.phase === 'capture_pending') {
@@ -415,6 +417,8 @@ function scheduleFinalizationJob(job: MeetingFinalizationJob, runtime: Finalizat
     // capture, and never throws back at the finalization path.
     if (!MAC_CAPTURE_ID.test(current.sessionId)) triggerMeetingMergeRun('g2_finalized')
     runtime.finalizationJobs.remove(current.sessionId)
+    // 6.78.0: finished, so its crash-loop count (lib/boot-resume.ts) is done.
+    bootResumeGate().forget('meeting_finalization', current.sessionId)
     finalizationRetryCounts.delete(key)
   }).catch(error => {
     const current = runtime.finalizationJobs.get(job.sessionId) ?? job
@@ -457,16 +461,28 @@ function scheduleFinalizationJob(job: MeetingFinalizationJob, runtime: Finalizat
 }
 
 /** Boot hook: replay post-save HQ/operations work whose response already
- * succeeded before a prior process exited. Safe to call more than once. */
-export function resumeMeetingFinalizationJobs(): void {
-  const finalizationJobs = new MeetingFinalizationJobStore()
+ * succeeded before a prior process exited. Safe to call more than once.
+ * 6.78.0: `sessionId` resumes one job (the maintenance Retry); the rest are seams for tests. */
+export function resumeMeetingFinalizationJobs(options: {
+  sessionId?: string
+  gate?: BootResumeGate
+  finalizationJobs?: MeetingFinalizationJobStore
+  schedule?: (job: MeetingFinalizationJob) => void
+} = {}): void {
+  const finalizationJobs = options.finalizationJobs ?? new MeetingFinalizationJobStore()
   finalizationJobs.reconcileCanonicalSidecars()
+  const gate = options.gate ?? bootResumeGate()
+  const schedule = options.schedule ?? ((job: MeetingFinalizationJob) => scheduleFinalizationJob(job, {
+    finalizationJobs,
+    runBatch: runMeetingBatchPipeline,
+    scheduleBackground: task => { void task },
+  }))
   for (const job of finalizationJobs.list()) {
-    scheduleFinalizationJob(job, {
-      finalizationJobs,
-      runBatch: runMeetingBatchPipeline,
-      scheduleBackground: task => { void task },
-    })
+    if (options.sessionId !== undefined && job.sessionId !== options.sessionId) continue
+    // 6.78.0: counted (durably) BEFORE it runs, so a job that kills the process is parked
+    // after RESTART_LIMIT crashed boots. A parked job is skipped and stays in its store.
+    if (!gate.admit('meeting_finalization', job.sessionId, job.phase)) continue
+    schedule(job)
   }
 }
 
@@ -808,7 +824,8 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
       if (alreadySaved) {
         finalizationJobs.reconcileCanonicalSidecars()
         const pendingJob = finalizationJobs.get(sessionId)
-        if (pendingJob) scheduleFinalizationJob(pendingJob, {
+        // 6.78.0: a replayed save never re-runs a job parked for crash-looping the server.
+        if (pendingJob && !bootResumeGate().isParked('meeting_finalization', sessionId)) scheduleFinalizationJob(pendingJob, {
           transcriptCleanup,
           finalizationJobs,
           runBatch,
@@ -973,6 +990,7 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
         markCanonicalFinalizationState(saved.sidecarPath, 'complete', false)
         if (!readMacRecording(sessionId)) triggerMeetingMergeRun('g2_finalized')
         finalizationJobs.remove(sessionId)
+        bootResumeGate().forget('meeting_finalization', sessionId)
         finalizationJob = null
       }
 

package/server/routes/query-jobs.ts

@@ -1,8 +1,10 @@
 import { Router, type Request, type Response } from 'express'
-import { durableQueryJobsEnabled } from '../lib/query-job-feature.js'
+import { durableQueryJobsEnabled, messagesSteerEnabled } from '../lib/query-job-feature.js'
 import {
   QueryJobCoordinator,
   QueryJobCoordinatorError,
+  QueryJobSteerLimitError,
+  QueryJobSteerUnknownError,
   queryJobErrorCode,
 } from '../lib/query-job-coordinator.js'
 import {
@@ -15,13 +17,16 @@ import {
   QueryJobNotTerminalError,
   QueryJobPersistenceError,
   QueryJobProviderOrphanFenceError,
+  QueryJobSteerConflictError,
   QueryJobStoreError,
+  type QueryJobSteerFrame,
   type QueryJobTrailFrame,
 } from '../lib/query-job-store.js'
 import {
   isTerminalQueryJobStatus,
   normalizeQueryJobError,
   parsePositiveInteger,
+  steerReadState,
   QueryJobValidationError,
   type QueryJobEvent,
   type QueryJobSnapshot,
@@ -33,6 +38,8 @@ export interface QueryJobsRouterOptions {
   enabled?: () => boolean
   heartbeatMs?: number
   prepareAdmission?: (raw: unknown) => unknown | Promise<unknown>
+  /** 6.78.0 Messages steer. Defaults to COS_MESSAGES_STEER (off). */
+  steerEnabled?: () => boolean
 }
 
 interface WireError {
@@ -104,13 +111,50 @@ function wantsTrail(req: Request): boolean {
   return req.query.trail === '1'
 }
 
-function forClient(job: QueryJobSnapshot, trail: boolean): QueryJobSnapshot {
+function forClient(job: QueryJobSnapshot, trail: boolean, steer = false): QueryJobSnapshot {
+  return steerView(trailView(job, trail), steer)
+}
+
+function trailView(job: QueryJobSnapshot, trail: boolean): QueryJobSnapshot {
   if (trail || !('trail' in job)) return job
   const { trail: _trail, ...rest } = job
   return rest as QueryJobSnapshot
 }
 
-function snapshotEvent(job: QueryJobSnapshot, reason: string, trail: boolean): Record<string, unknown> {
+/**
+ * 6.78.0: STEERS ARE OPT-IN, like the trail. Without `?steer=1` a snapshot carries no `steers`
+ * (exactly the 6.77.0 shape); with it, each steer reads as a client should see it now
+ * (`not_applied` for one an ended job did not get to apply).
+ */
+function wantsSteer(req: Request): boolean {
+  return req.query.steer === '1'
+}
+
+function steerView(job: QueryJobSnapshot, steer: boolean): QueryJobSnapshot {
+  if (!('steers' in job)) return job
+  const { steers, ...rest } = job
+  if (!steer || !steers) return rest as QueryJobSnapshot
+  return { ...rest, steers: steers.map(item => ({ ...item, state: steerReadState(item.state, job.status) })) } as QueryJobSnapshot
+}
+
+/** A steer frame on the wire (contract messages-steer.json `sse_frame`): no `id:` line, no eventSeq. */
+function writeSteer(res: Response, frame: Pick<QueryJobSteerFrame, 'steerId' | 'state' | 'at'>): void {
+  res.write(`event: steer\ndata: ${JSON.stringify({ steerId: frame.steerId, state: frame.state, at: frame.at })}\n\n`)
+}
+
+/** The steer routes' error bodies (contract messages-steer.json `errors`). */
+function steerWireError(error: unknown): { status: number; body: WireError } {
+  if (error instanceof QueryJobSteerConflictError) return { status: 409, body: { error: { code: error.code, message: error.message } } }
+  if (error instanceof QueryJobSteerLimitError) return { status: 429, body: { error: { code: error.code, message: error.message, retryable: false } } }
+  if (error instanceof QueryJobSteerUnknownError) return { status: 404, body: { error: { code: error.code, message: error.message } } }
+  if (error instanceof QueryJobNotFoundError) return { status: 404, body: { error: { code: 'query_job_not_found', message: 'Query job not found' } } }
+  if (error instanceof QueryJobGenerationMismatchError) return { status: 409, body: { error: { code: 'query_job_generation_mismatch', message: 'Generation mismatch' } } }
+  return wireError(error)
+}
+
+const STEER_DISABLED: WireError = { error: { code: 'steer_disabled', message: 'Steering is off on this Mac.' } }
+
+function snapshotEvent(job: QueryJobSnapshot, reason: string, trail: boolean, steer = false): Record<string, unknown> {
   return {
     type: 'snapshot',
     eventSeq: job.eventSeq,
@@ -119,7 +163,7 @@ function snapshotEvent(job: QueryJobSnapshot, reason: string, trail: boolean): R
     generation: job.generation,
     status: job.status,
     at: job.updatedAt,
-    data: { reason, job: forClient(job, trail) },
+    data: { reason, job: forClient(job, trail, steer) },
   }
 }
 
@@ -129,6 +173,9 @@ export function createQueryJobsRouter(
 ): Router {
   const router = Router()
   const enabled = options.enabled ?? durableQueryJobsEnabled
+  const steerEnabled = options.steerEnabled ?? messagesSteerEnabled
+  // Steers reach a client only when it asks AND the switch is on.
+  const steerRead = (req: Request) => wantsSteer(req) && steerEnabled()
   const heartbeatMs = Math.max(1_000, options.heartbeatMs ?? 15_000)
 
   router.post('/query-jobs', async (req, res) => {
@@ -143,7 +190,7 @@ export function createQueryJobsRouter(
     try {
       const prepared = options.prepareAdmission ? await options.prepareAdmission(req.body) : req.body
       const admission = await coordinator.submit(prepared)
-      return res.status(202).json({ job: forClient(admission.job, wantsTrail(req)) })
+      return res.status(202).json({ job: forClient(admission.job, wantsTrail(req), steerRead(req)) })
     } catch (error) {
       const wire = wireError(error)
       return res.status(wire.status).json(wire.body)
@@ -159,19 +206,32 @@ export function createQueryJobsRouter(
       const generation = requiredGeneration(req.query.generation)
       const job = await coordinator.getByClientGeneration(clientJobId, generation)
       if (!job) throw new QueryJobNotFoundError(`${clientJobId}:${generation}`)
-      return res.json({ job: forClient(job, wantsTrail(req)) })
+      return res.json({ job: forClient(job, wantsTrail(req), steerRead(req)) })
     } catch (error) {
       const wire = wireError(error)
       return res.status(wire.status).json(wire.body)
     }
   })
 
+  // 6.78.0: a steer by its id and lineage, from the newest generation that holds it. Ahead of
+  // the dynamic job routes, like by-client.
+  router.get('/query-jobs/steers/:steerId', async (req, res) => {
+    if (!enabled() || !steerEnabled()) return res.status(404).json(STEER_DISABLED)
+    try {
+      const clientJobId = validClientJobId(req.query.clientJobId)
+      return res.json(await coordinator.steerStatus(clientJobId, req.params.steerId))
+    } catch (error) {
+      const wire = steerWireError(error)
+      return res.status(wire.status).json(wire.body)
+    }
+  })
+
   router.get('/query-jobs/:jobId', async (req, res) => {
     try {
       const jobId = validJobId(req.params.jobId)
       const generation = requiredGeneration(req.query.generation)
       const job = await coordinator.getSnapshot(jobId, generation)
-      return res.json({ job: forClient(job, wantsTrail(req)) })
+      return res.json({ job: forClient(job, wantsTrail(req), steerRead(req)) })
     } catch (error) {
       const wire = wireError(error)
       return res.status(wire.status).json(wire.body)
@@ -188,6 +248,9 @@ export function createQueryJobsRouter(
     const pendingLive: QueryJobEvent[] = []
     const pendingTrail: QueryJobTrailFrame[] = []
     const trail = wantsTrail(req)
+    // 6.78.0: `?steer=1` adds `steer` frames; without it the stream is the 6.77.0 one.
+    const steer = steerRead(req)
+    const pendingSteer: QueryJobSteerFrame[] = []
     const writeTrail = (frame: QueryJobTrailFrame) => {
       if (closed || frame.trailSeq <= sentTrailSeq) return
       sentTrailSeq = frame.trailSeq
@@ -243,8 +306,22 @@ export function createQueryJobsRouter(
         writeSse(res, event.type, event as unknown as Record<string, unknown>)
         if (isTerminalQueryJobStatus(event.status)) finish()
       }
+      const steerSubscription = steer
+        ? {
+          listener: (frame: QueryJobSteerFrame) => {
+            if (closed) return
+            if (!streamReady) {
+              pendingSteer.push(frame)
+              return
+            }
+            writeSteer(res, frame)
+          },
+        }
+        : undefined
       // Without `?trail=1` the call is exactly the 6.51.0 one.
-      const subscription = trailSubscription
+      const subscription = steerSubscription
+        ? await coordinator.subscribe(jobId, generation, after, onEvent, trailSubscription, steerSubscription)
+        : trailSubscription
         ? await coordinator.subscribe(jobId, generation, after, onEvent, trailSubscription)
         : await coordinator.subscribe(jobId, generation, after, onEvent)
       unsubscribe = subscription.unsubscribe
@@ -263,7 +340,10 @@ export function createQueryJobsRouter(
       res.flushHeaders()
       res.write(': keepalive\n\n')
 
-      const { replay, trailReplay } = subscription
+      const { replay, trailReplay, steerReplay } = subscription
+      // 6.78.0: every steer as it reads now, before any job event (a state only moves forward,
+      // so a client that saw some of these simply sees them again).
+      if (steer && steerReplay) for (const item of steerReplay) writeSteer(res, item)
       // The trail first: every entry precedes the terminal event that may end this stream.
       if (trail && trailReplay) {
         if (trailReplay.gap) {
@@ -289,7 +369,7 @@ export function createQueryJobsRouter(
         }
       }
       if (replay.gap) {
-        writeSse(res, 'snapshot', snapshotEvent(replay.snapshot, replay.reason ?? 'replay_gap', trail))
+        writeSse(res, 'snapshot', snapshotEvent(replay.snapshot, replay.reason ?? 'replay_gap', trail, steer))
         sentSeq = replay.snapshot.eventSeq
       } else {
         for (const event of replay.events) {
@@ -301,7 +381,7 @@ export function createQueryJobsRouter(
 
       const replayTerminal = replay.events.some(event => isTerminalQueryJobStatus(event.status))
       if (isTerminalQueryJobStatus(replay.snapshot.status) && !replayTerminal && !replay.gap) {
-        writeSse(res, 'snapshot', snapshotEvent(replay.snapshot, 'terminal_snapshot', trail))
+        writeSse(res, 'snapshot', snapshotEvent(replay.snapshot, 'terminal_snapshot', trail, steer))
         sentSeq = replay.snapshot.eventSeq
       }
       if (isTerminalQueryJobStatus(replay.snapshot.status)) {
@@ -312,6 +392,8 @@ export function createQueryJobsRouter(
       streamReady = true
       for (const frame of pendingTrail) writeTrail(frame)
       pendingTrail.length = 0
+      for (const frame of pendingSteer) writeSteer(res, frame)
+      pendingSteer.length = 0
       for (const event of pendingLive) {
         if (closed || event.eventSeq <= sentSeq) continue
         sentSeq = event.eventSeq
@@ -344,7 +426,7 @@ export function createQueryJobsRouter(
       const jobId = validJobId(req.params.jobId)
       const generation = requiredGeneration(req.body?.generation)
       const { job } = await coordinator.cancel(jobId, generation)
-      return res.json({ job: forClient(job, wantsTrail(req)) })
+      return res.json({ job: forClient(job, wantsTrail(req), steerRead(req)) })
     } catch (error) {
       const wire = wireError(error)
       return res.status(wire.status).json(wire.body)
@@ -356,12 +438,24 @@ export function createQueryJobsRouter(
       const jobId = validJobId(req.params.jobId)
       const generation = requiredGeneration(req.body?.generation)
       const job = await coordinator.acknowledge(jobId, generation)
-      return res.json({ job: forClient(job, wantsTrail(req)) })
+      return res.json({ job: forClient(job, wantsTrail(req), steerRead(req)) })
     } catch (error) {
       const wire = wireError(error)
       return res.status(wire.status).json(wire.body)
     }
   })
 
+  // 6.78.0 Messages steer (contract messages-steer.json). 404 steer_disabled when off.
+  router.post('/query-jobs/:jobId/steer', async (req, res) => {
+    if (!enabled() || !steerEnabled()) return res.status(404).json(STEER_DISABLED)
+    try {
+      const jobId = validJobId(req.params.jobId)
+      return res.json(await coordinator.steer(jobId, req.body))
+    } catch (error) {
+      const wire = steerWireError(error)
+      return res.status(wire.status).json(wire.body)
+    }
+  })
+
   return router
 }

package/server/routes/recovery.ts

@@ -2,6 +2,7 @@ import { Router } from 'express'
 import { readFileSync } from 'node:fs'
 import { resolve } from 'node:path'
 import { atomicWriteFileSync } from '../lib/atomic-fs.js'
+import { bootResumeGate } from '../lib/boot-resume.js'
 import { scheduleManagedServerRestart } from '../lib/managed-restart.js'
 import { isManagedRuntime } from '../lib/managed-runtime.js'
 import { acquireMaintenance, getRecoveryActivityStatus } from '../lib/recovery-activity.js'
@@ -72,6 +73,9 @@ recoveryRouter.post('/recovery/server/restart', (_req, res) => {
   // SuccessfulExit:false, so a clean exit leaves the service STOPPED and the
   // phone shows "Restart requested; reconnect is taking longer than expected."
   scheduleManagedServerRestart({
+    // 6.78.0 (P7 S4): a requested restart is a clean exit, whichever way the process ends
+    // (SIGTERM from kickstart -k, or the 2 s exit(1) fallback): it never counts as a crash.
+    beforeRestart: () => bootResumeGate().markCleanShutdown(),
     onError: (error) => {
       gate.release()
       console.error('[recovery] Failed to schedule managed server restart:', error)