@pikku/core 0.12.104 → 0.12.106

package/src/services/credential-wire-service.test.ts

@@ -122,6 +122,69 @@ describe('PikkuCredentialWireService', () => {
     assert.deepStrictEqual(r1, r2)
   })
 
+  /**
+   * The regression the `loaded` flag was written to avoid.
+   *
+   * `get` answers synchronously once `loaded` is set, so setting the flag
+   * before the fetch resolved gave every caller that arrived during the load a
+   * confident `null` for a credential that exists. `getAll` hid it: it hands
+   * back the live `credentials` object, which is filled in before anyone
+   * asserts on it, so only a reader that copies a VALUE out sees the gap.
+   */
+  test('should not answer null while a lazy load is still in flight', async () => {
+    const credService = mockCredentialService({ stripe: { key: 'sk_live' } })
+    const wire: PikkuWire = { session: { userId: 'user-1' } as any }
+    const service = new PikkuCredentialWireService(credService, wire)
+
+    // Issued in one tick, exactly as `Promise.all` over sibling RPC calls does.
+    const results = await Promise.all([
+      service.get('stripe'),
+      service.get('stripe'),
+      service.get('stripe'),
+    ])
+
+    for (const [index, result] of results.entries()) {
+      assert.deepStrictEqual(
+        result,
+        { key: 'sk_live' },
+        `concurrent reader ${index} lost the credential`
+      )
+    }
+  })
+
+  test('should not answer null during a slow lazy load', async () => {
+    const credService: CredentialService = {
+      get: async () => null,
+      set: async () => {},
+      delete: async () => {},
+      has: async () => true,
+      getAll: async () => {
+        await new Promise((resolve) => setTimeout(resolve, 10))
+        return { stripe: { key: 'sk_live' } }
+      },
+    }
+    const wire: PikkuWire = { session: { userId: 'user-1' } as any }
+    const service = new PikkuCredentialWireService(credService, wire)
+
+    const first = service.get('stripe')
+    const second = service.get('stripe')
+    assert.deepStrictEqual(await first, { key: 'sk_live' })
+    assert.deepStrictEqual(await second, { key: 'sk_live' })
+  })
+
+  test('should still take the sync path when there is nothing to load', async () => {
+    // Both early returns are settled states, so they must set `loaded` too.
+    const service = new PikkuCredentialWireService(
+      mockCredentialService({ stripe: { key: '1' } }),
+      {}
+    )
+    await service.getAll()
+    assert.ok(
+      !(service.getAll() instanceof Promise),
+      'expected sync return after an unresolvable user'
+    )
+  })
+
   test('getScoped should only return allowed names', async () => {
     const credService = mockCredentialService({
       stripe: { key: '1' },

package/src/services/credential-wire-service.ts

@@ -62,17 +62,35 @@ export class PikkuCredentialWireService {
     return this.loadPromise
   }
 
+  /**
+   * `loaded` means "the credentials are in hand", never "a load has started".
+   *
+   * The distinction matters because `get`, `getAll` and `getScoped` all answer
+   * SYNCHRONOUSLY once it is set. Setting it before the await let every caller
+   * that arrived while the fetch was still in flight take that synchronous path
+   * over an empty map and receive `null` for a credential that exists — the
+   * first of N concurrent readers got the value and the rest silently did not.
+   * `loadPromise` is what guards against loading twice; this flag is only ever
+   * about whether the fast path is safe to take.
+   *
+   * `finally` rather than a trailing assignment: the two early returns (no
+   * service or wire, no resolvable user) are settled states as well — there is
+   * nothing to fetch — so they should reach the fast path too.
+   */
   private async doLoad(): Promise<void> {
-    this.loaded = true
-    if (!this.credentialService || !this.wire) return
-    const userId = defaultPikkuUserIdResolver(this.wire)
-    if (!userId) return
-    this.wire.pikkuUserId = userId
-    const allCreds = await this.credentialService.getAll(userId)
-    for (const [name, value] of Object.entries(allCreds)) {
-      if (!(name in this.credentials)) {
-        this.credentials[name] = value
+    try {
+      if (!this.credentialService || !this.wire) return
+      const userId = defaultPikkuUserIdResolver(this.wire)
+      if (!userId) return
+      this.wire.pikkuUserId = userId
+      const allCreds = await this.credentialService.getAll(userId)
+      for (const [name, value] of Object.entries(allCreds)) {
+        if (!(name in this.credentials)) {
+          this.credentials[name] = value
+        }
       }
+    } finally {
+      this.loaded = true
     }
   }
 }

package/src/services/index.ts

@@ -18,6 +18,16 @@ export {
   QueueWebhookService,
   pikkuWebhookWorkerFunc,
 } from './queue-webhook-service.js'
+export {
+  REMOTE_JOBS_SECRET_HEADER,
+  REMOTE_JOBS_SECRET_VARIABLE,
+  REMOTE_QUEUE_JOB_PATH,
+  REMOTE_SCHEDULER_JOB_PATH,
+  pikkuRemoteQueueJobFunc,
+  pikkuRemoteScheduledJobFunc,
+  type RemoteQueueJobData,
+  type RemoteScheduledJobData,
+} from './remote-jobs.js'
 export { InMemoryQueueService } from './in-memory-queue-service.js'
 export { InMemoryTriggerService } from './in-memory-trigger-service.js'
 export { InMemoryAgentRunStateService } from './in-memory-agent-run-state-service.js'

package/src/services/remote-jobs.test.ts

@@ -0,0 +1,106 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import { BadRequestError, UnauthorizedError } from '../errors/errors.js'
+import { resetPikkuState, setSingletonServices } from '../pikku-state.js'
+import { remoteJobsSecret } from '../middleware/remote-jobs-secret.js'
+import {
+  REMOTE_JOBS_SECRET_HEADER,
+  pikkuRemoteScheduledJobFunc,
+} from './remote-jobs.js'
+
+const silentLogger = {
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+  debug: () => {},
+} as any
+
+const wire = (header?: string) =>
+  ({
+    http: { request: { header: () => header ?? null } },
+  }) as any
+
+const services = (secret?: string) =>
+  ({ variables: { get: async () => secret } }) as any
+
+describe('remoteJobsSecret', () => {
+  test('lets a caller holding the secret through', async () => {
+    let reached = false
+    await remoteJobsSecret(services('s3cret'), wire('s3cret'), async () => {
+      reached = true
+    })
+    assert.equal(reached, true)
+  })
+
+  test('rejects a caller presenting the wrong secret', async () => {
+    await assert.rejects(
+      remoteJobsSecret(services('s3cret'), wire('nope'), async () => {}),
+      UnauthorizedError
+    )
+  })
+
+  test('rejects when the deployment configured no secret at all', async () => {
+    await assert.rejects(
+      remoteJobsSecret(services(undefined), wire('anything'), async () => {}),
+      UnauthorizedError
+    )
+  })
+
+  test('rejects a caller presenting no header', async () => {
+    await assert.rejects(
+      remoteJobsSecret(services('s3cret'), wire(), async () => {}),
+      UnauthorizedError
+    )
+  })
+
+  test('reads the header the dispatchers already send', async () => {
+    const seen: string[] = []
+    await assert.rejects(
+      remoteJobsSecret(
+        services('s3cret'),
+        {
+          http: {
+            request: {
+              header: (name: string) => {
+                seen.push(name)
+                return null
+              },
+            },
+          },
+        } as any,
+        async () => {}
+      ),
+      UnauthorizedError
+    )
+    assert.deepEqual(seen, [REMOTE_JOBS_SECRET_HEADER])
+  })
+
+  test('awaits a variables service that resolves asynchronously', async () => {
+    let reached = false
+    await remoteJobsSecret(
+      {
+        variables: {
+          get: () => new Promise((resolve) => setTimeout(() => resolve('s3cret'), 1)),
+        },
+      } as any,
+      wire('s3cret'),
+      async () => {
+        reached = true
+      }
+    )
+    assert.equal(reached, true)
+  })
+})
+
+describe('pikkuRemoteScheduledJobFunc', () => {
+  test('acks a task this deployment does not have rather than asking for a retry', async () => {
+    resetPikkuState()
+    setSingletonServices({ logger: silentLogger } as any)
+    await assert.rejects(
+      pikkuRemoteScheduledJobFunc({ logger: silentLogger }, {
+        taskName: 'not-a-task',
+      }),
+      BadRequestError
+    )
+  })
+})

package/src/services/remote-jobs.ts

@@ -0,0 +1,92 @@
+import { BadRequestError, PikkuMissingMetaError } from '../errors/errors.js'
+import { QueueJobDiscardedError, runQueueJob } from '../wirings/queue/queue-runner.js'
+import type { QueueJob, QueueJobStatus } from '../wirings/queue/queue.types.js'
+import {
+  ScheduledTaskNotFoundError,
+  runScheduledTask,
+} from '../wirings/scheduler/scheduler-runner.js'
+import type { Logger } from './logger.js'
+
+/** The variable holding the shared secret every remote job must present. */
+export const REMOTE_JOBS_SECRET_VARIABLE = 'PIKKU_DISPATCH_SECRET'
+
+export const REMOTE_JOBS_SECRET_HEADER = 'x-pikku-dispatch'
+
+export const REMOTE_QUEUE_JOB_PATH = '/__pikku/queue-job'
+
+export const REMOTE_SCHEDULER_JOB_PATH = '/__pikku/scheduler-job'
+
+export interface RemoteQueueJobData {
+  queueName: string
+  data?: unknown
+  jobId?: string
+  traceId?: string
+}
+
+export interface RemoteScheduledJobData {
+  taskName: string
+}
+
+/**
+ * A dispatched job is delivered, not owned: the sender retries, so a permanent
+ * failure has to read differently from a transient one. An unknown worker or a
+ * discarded job answers 4xx so the sender stops; anything else propagates and
+ * is delivered again.
+ */
+export async function pikkuRemoteQueueJobFunc(
+  { logger }: { logger: Logger },
+  { queueName, data, jobId, traceId }: RemoteQueueJobData
+): Promise<void> {
+  const job: QueueJob = {
+    queueName,
+    data,
+    id: jobId ?? traceId ?? `remote-${Date.now()}`,
+    status: async (): Promise<QueueJobStatus> => 'active',
+    metadata: () => ({
+      processedAt: new Date(),
+      attemptsMade: 0,
+      maxAttempts: undefined,
+      result: undefined,
+      progress: 0,
+      createdAt: new Date(),
+      completedAt: undefined,
+      failedAt: undefined,
+      error: undefined,
+    }),
+    waitForCompletion: async () => {
+      throw new Error('remote jobs do not support waitForCompletion')
+    },
+  }
+
+  try {
+    await runQueueJob({ job, traceId })
+  } catch (error) {
+    if (
+      error instanceof PikkuMissingMetaError ||
+      error instanceof QueueJobDiscardedError
+    ) {
+      throw new BadRequestError(error.message)
+    }
+    logger.error(
+      `remote-jobs: queue '${queueName}' failed: ${error instanceof Error ? error.message : String(error)}`
+    )
+    throw error
+  }
+}
+
+export async function pikkuRemoteScheduledJobFunc(
+  { logger }: { logger: Logger },
+  { taskName }: RemoteScheduledJobData
+): Promise<void> {
+  try {
+    await runScheduledTask({ name: taskName, traceId: `cron-${taskName}` })
+  } catch (error) {
+    if (error instanceof ScheduledTaskNotFoundError) {
+      throw new BadRequestError(error.message)
+    }
+    logger.error(
+      `remote-jobs: task '${taskName}' failed: ${error instanceof Error ? error.message : String(error)}`
+    )
+    throw error
+  }
+}

package/src/wirings/scheduler/index.ts

@@ -1,4 +1,5 @@
 export {
+  ScheduledTaskNotFoundError,
   runScheduledTask,
   wireScheduler,
   getScheduledTasks,

package/src/wirings/scheduler/scheduler-runner.ts

@@ -52,7 +52,7 @@ export const wireScheduler = <
   tasks.set(scheduledTask.name, scheduledTask)
 }
 
-class ScheduledTaskNotFoundError extends PikkuError {
+export class ScheduledTaskNotFoundError extends PikkuError {
   constructor(title: string) {
     super(`Scheduled task not found: ${title}`)
   }