@pikku/core 0.12.70 → 0.12.72

package/dist/wirings/workflow/workflow.types.d.ts

@@ -182,11 +182,6 @@ export interface WorkflowRunService {
         graph: any;
         source: string;
     } | null>;
-    getAIGeneratedWorkflows(agentName?: string): Promise<Array<{
-        workflowName: string;
-        graphHash: string;
-        graph: any;
-    }>>;
     deleteRun(id: string): Promise<boolean>;
 }
 /**
@@ -367,7 +362,7 @@ export interface WorkflowRuntimeMeta {
     /** Pikku function name (for execution) */
     pikkuFuncId: string;
     /** Source type: 'dsl' (serializable), 'complex' (has inline steps), 'graph', 'scenario' (complex + actor steps) */
-    source: 'dsl' | 'complex' | 'graph' | 'dynamic-workflow' | 'scenario';
+    source: 'dsl' | 'complex' | 'graph' | 'scenario';
     /** Optional description */
     description?: string;
     /** Tags for organization */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/core",
-  "version": "0.12.70",
+  "version": "0.12.72",
   "description": "The Pikku runtime — functions, wirings, services, middleware and types",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",
@@ -55,7 +55,6 @@
     "./services/istanbul-coverage": "./dist/services/istanbul-coverage-service.js",
     "./services/local-content": "./dist/services/local-content.js",
     "./services/temporary-file-service": "./dist/services/temporary-file-service.js",
-    "./services/gopass-secrets": "./dist/services/gopass-secrets.js",
     "./crypto-utils": "./dist/crypto-utils.js",
     "./hmac": "./dist/utils/hmac.js",
     "./internal": "./dist/internal.js",

package/src/function/function-runner.test.ts

@@ -7,6 +7,7 @@ import {
   runPikkuFunc,
 } from './function-runner.js'
 import { addGlobalPermission, addTagMiddleware } from '../index.js'
+import { addSchema } from '../schema.js'
 import { resetPikkuState, pikkuState } from '../pikku-state.js'
 import type { CoreServices, CorePikkuMiddleware } from '../types/core.types.js'
 import type { CorePermissionGroup } from './functions.types.js'
@@ -1494,4 +1495,78 @@ describe('runPikkuFunc - scopes', () => {
       MissingScopeError
     )
   })
+
+  describe('schema defaults', () => {
+    const addPagedFunction = (funcName: string, seen: { data?: any }) => {
+      addSchema('PagedInput', {
+        properties: {
+          page: { type: 'number', default: 1 },
+          limit: { type: 'number', default: 50 },
+        },
+      })
+      addTestFunction(funcName, {
+        func: async (_services: any, data: any) => {
+          seen.data = data
+          return 'ok'
+        },
+      })
+      pikkuState(null, 'function', 'meta')[funcName]!.inputSchemaName =
+        'PagedInput'
+    }
+
+    // The whole point of the fix: `{}` validates because a defaulted property
+    // is not `required`, so without this the body reads `undefined` for a value
+    // its generated type declares as a number.
+    test('a function called with {} receives its schema defaults', async () => {
+      const seen: { data?: any } = {}
+      addPagedFunction('paged', seen)
+
+      await runPikkuFunc('rpc', Math.random().toString(), 'paged', {
+        singletonServices: mockSingletonServices,
+        getAllServices: () => mockServices,
+        data: () => ({}),
+        auth: false,
+        wire: {},
+      })
+
+      assert.equal(seen.data.page, 1)
+      assert.equal(seen.data.limit, 50)
+    })
+
+    // `coerceDataFromSchema` is not passed here, exactly as a direct RPC
+    // invocation leaves it. Defaults belong to the schema rather than to the
+    // transport, so they must not ride on that flag — and a call made with no
+    // arguments at all still has to get them.
+    test('defaults apply with coercion off and no data', async () => {
+      const seen: { data?: any } = {}
+      addPagedFunction('pagedNoCoerce', seen)
+
+      await runPikkuFunc('rpc', Math.random().toString(), 'pagedNoCoerce', {
+        singletonServices: mockSingletonServices,
+        getAllServices: () => mockServices,
+        data: () => undefined,
+        auth: false,
+        wire: {},
+      })
+
+      assert.equal(seen.data.page, 1)
+      assert.equal(seen.data.limit, 50)
+    })
+
+    test('a supplied value survives', async () => {
+      const seen: { data?: any } = {}
+      addPagedFunction('pagedSupplied', seen)
+
+      await runPikkuFunc('rpc', Math.random().toString(), 'pagedSupplied', {
+        singletonServices: mockSingletonServices,
+        getAllServices: () => mockServices,
+        data: () => ({ page: 3 }),
+        auth: false,
+        wire: {},
+      })
+
+      assert.equal(seen.data.page, 3)
+      assert.equal(seen.data.limit, 50)
+    })
+  })
 })

package/src/function/function-runner.ts

@@ -5,7 +5,11 @@ import {
 } from '../wirings/channel/channel-middleware-runner.js'
 import { runPermissions } from '../permissions.js'
 import { pikkuState } from '../pikku-state.js'
-import { coerceTopLevelDataFromSchema, validateSchema } from '../schema.js'
+import {
+  applyDefaultsFromSchema,
+  coerceTopLevelDataFromSchema,
+  validateSchema,
+} from '../schema.js'
 import type {
   CoreUserSession,
   CorePikkuMiddleware,
@@ -293,11 +297,20 @@ export const runPikkuFunc = async <In = any, Out = any>(
     verifyScopes(funcConfig.scopes ?? funcMeta.scopes, session)
 
     // Evaluate the data from the lazy function
-    const actualData = await data()
+    let actualData = await data()
 
     // Validate and coerce data if schema is defined
     const inputSchemaName = funcMeta.inputSchemaName
     if (inputSchemaName) {
+      // Fill in schema defaults before anything reads the data. Unconditional:
+      // a default belongs to the schema, not to the transport the call arrived
+      // on. Runs before coercion so a defaulted value and a supplied one are
+      // treated identically from here on.
+      actualData = applyDefaultsFromSchema(
+        inputSchemaName,
+        actualData,
+        packageName
+      )
       // Coerce (top level) data types before validation (e.g. string→array, string→date)
       if (coerceDataFromSchema) {
         coerceTopLevelDataFromSchema(inputSchemaName, actualData, packageName)

package/src/gopass-secrets-removed.test.ts

@@ -0,0 +1,51 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import { existsSync, readFileSync, readdirSync } from 'node:fs'
+import { fileURLToPath } from 'node:url'
+import { join, dirname } from 'node:path'
+
+const srcRoot = dirname(fileURLToPath(import.meta.url))
+const packageRoot = join(srcRoot, '..')
+
+const collectSourceFiles = (directory: string): string[] =>
+  readdirSync(directory, { withFileTypes: true }).flatMap((entry) => {
+    const entryPath = join(directory, entry.name)
+    if (entry.isDirectory()) {
+      return entry.name === 'dist' ? [] : collectSourceFiles(entryPath)
+    }
+    return /\.(ts|js|mts|cts)$/.test(entry.name) ? [entryPath] : []
+  })
+
+describe('gopass secret service removal', () => {
+  test('the gopass-secrets module no longer exists', () => {
+    assert.equal(
+      existsSync(join(srcRoot, 'services/gopass-secrets.ts')),
+      false,
+      'services/gopass-secrets.ts was reintroduced'
+    )
+  })
+
+  test('package.json no longer exports ./services/gopass-secrets', () => {
+    const packageJson = JSON.parse(
+      readFileSync(join(packageRoot, 'package.json'), 'utf-8')
+    )
+    assert.equal(
+      packageJson.exports['./services/gopass-secrets'],
+      undefined,
+      './services/gopass-secrets export was reintroduced'
+    )
+  })
+
+  test('no source file references gopass', () => {
+    const offenders = collectSourceFiles(srcRoot).filter(
+      (file) =>
+        file !== fileURLToPath(import.meta.url) &&
+        /gopass/i.test(readFileSync(file, 'utf-8'))
+    )
+    assert.deepEqual(
+      offenders,
+      [],
+      `gopass references found in:\n${offenders.join('\n')}`
+    )
+  })
+})

package/src/handle-error.test.ts

@@ -1,11 +1,12 @@
 import { describe, test, beforeEach } from 'node:test'
 import assert from 'node:assert'
 import { handleHTTPError } from './handle-error.js'
-import { addError } from './errors/error-handler.js'
+import { addError, PikkuError } from './errors/error-handler.js'
 import {
   NotFoundError,
   BadRequestError,
   ForbiddenError,
+  MissingScopeError,
 } from './errors/errors.js'
 import { resetPikkuState } from './pikku-state.js'
 
@@ -399,6 +400,112 @@ describe('handleHTTPError', () => {
     assert.deepStrictEqual(http._state.jsonBody, { errorId: 'tracker-str' })
   })
 
+  test('should not leak the payload of a registered server error', () => {
+    class LeakyInternalError extends PikkuError {}
+    addError(LeakyInternalError, {
+      status: 500,
+      message: 'Something went wrong',
+    })
+
+    const logger = createMockLogger()
+    const http = createMockHTTP()
+    const error = new LeakyInternalError(
+      'connection to 10.0.0.1 refused'
+    ) as any
+    error.payload = { connectionString: 'postgres://user:hunter2@10.0.0.1' }
+
+    handleHTTPError(
+      error,
+      http as any,
+      'tracker-leak',
+      logger as any,
+      [],
+      true,
+      false
+    )
+
+    assert.strictEqual(http._state.statusCode, 500)
+    assert.strictEqual(http._state.jsonBody.payload, undefined)
+  })
+
+  test('should not leak the raw message of a registered server error', () => {
+    class InternalMessageError extends PikkuError {}
+    addError(InternalMessageError, {
+      status: 500,
+      message: 'Something went wrong',
+    })
+
+    const logger = createMockLogger()
+    const http = createMockHTTP()
+    const error = new InternalMessageError('secret internal detail')
+
+    handleHTTPError(
+      error,
+      http as any,
+      'tracker-msg',
+      logger as any,
+      [],
+      true,
+      false
+    )
+
+    assert.strictEqual(http._state.statusCode, 500)
+    assert.strictEqual(http._state.jsonBody.message, 'Something went wrong')
+  })
+
+  test('should keep the payload of a client facing error', () => {
+    const logger = createMockLogger()
+    const http = createMockHTTP()
+    const error = new MissingScopeError('admin:write')
+
+    handleHTTPError(
+      error,
+      http as any,
+      'tracker-scope',
+      logger as any,
+      [],
+      true,
+      false
+    )
+
+    assert.strictEqual(http._state.statusCode, 403)
+    assert.deepStrictEqual(http._state.jsonBody.payload, {
+      error: 'missing_scope',
+      scope: 'admin:write',
+    })
+    assert.strictEqual(
+      http._state.jsonBody.message,
+      'Missing required scope: admin:write'
+    )
+  })
+
+  test('should expose server error details when exposeErrors is true', () => {
+    class ExposableInternalError extends PikkuError {}
+    addError(ExposableInternalError, {
+      status: 500,
+      message: 'Something went wrong',
+    })
+
+    const logger = createMockLogger()
+    const http = createMockHTTP()
+    const error = new ExposableInternalError('internal detail') as any
+    error.payload = { debug: true }
+
+    handleHTTPError(
+      error,
+      http as any,
+      'tracker-expose-known',
+      logger as any,
+      [],
+      true,
+      false,
+      true
+    )
+
+    assert.strictEqual(http._state.jsonBody.message, 'internal detail')
+    assert.deepStrictEqual(http._state.jsonBody.payload, { debug: true })
+  })
+
   test('should handle warning log without trackerId', () => {
     const logger = createMockLogger()
     const http = createMockHTTP()

package/src/handle-error.ts

@@ -14,6 +14,8 @@ import type { PikkuHTTP } from './wirings/http/http.types.js'
  * @param {number[]} logWarningsForStatusCodes - HTTP status codes to log as warnings
  * @param {boolean} respondWith404 - Whether to respond with 404 for NotFoundError
  * @param {boolean} bubbleError - Whether to throw the error after handling
+ * @param {boolean} exposeErrors - Whether to include internal error details (message, stack and
+ * payload of 5xx errors) in the response body. Ignored in production.
  */
 export const handleHTTPError = (
   e: any,
@@ -33,15 +35,21 @@ export const handleHTTPError = (
   // Get appropriate error response
   const errorResponse = getErrorResponse(e)
   if (errorResponse != null) {
+    const clientFacing =
+      errorResponse.status < 500 || (exposeErrors && !isProduction())
+
     // Set status and response body
     http?.response?.status(errorResponse.status)
     http?.response?.json({
       name: e instanceof Error ? e.name : undefined,
       message:
-        e instanceof Error && e.message && e.message !== 'An error occurred'
+        clientFacing &&
+        e instanceof Error &&
+        e.message &&
+        e.message !== 'An error occurred'
           ? e.message
           : errorResponse.message,
-      payload: (e as any).payload,
+      payload: clientFacing ? (e as any).payload : undefined,
       errorId: traceId,
     })
 

package/src/schema.test.ts

@@ -3,6 +3,7 @@ import * as assert from 'assert'
 import {
   addSchema,
   getSchema,
+  applyDefaultsFromSchema,
   coerceTopLevelDataFromSchema,
   validateSchema,
   compileAllSchemas,
@@ -111,6 +112,108 @@ describe('Schema', () => {
     })
   })
 
+  describe('applyDefaultsFromSchema', () => {
+    before(() => {
+      addSchema('defaultsSchema', {
+        properties: {
+          page: { type: 'number', default: 1 },
+          limit: { type: 'number', default: 50 },
+          addons: { type: 'array', default: [] },
+          mode: { type: 'string', default: 'manual' },
+          enabled: { type: 'boolean', default: true },
+          name: { type: 'string' },
+        },
+      })
+
+      addSchema('noDefaultsSchema', {
+        properties: { name: { type: 'string' } },
+      })
+
+      addSchema('booleanPropSchema', {
+        properties: { isActive: true, page: { type: 'number', default: 1 } },
+      })
+    })
+
+    test('should fill in absent properties from their default', () => {
+      const data = applyDefaultsFromSchema('defaultsSchema', {})
+      assert.strictEqual(data.page, 1)
+      assert.strictEqual(data.limit, 50)
+      assert.strictEqual(data.mode, 'manual')
+      assert.strictEqual(data.enabled, true)
+    })
+
+    test('should not overwrite a supplied value', () => {
+      const data = applyDefaultsFromSchema('defaultsSchema', {
+        page: 7,
+        limit: 200,
+      })
+      assert.strictEqual(data.page, 7)
+      assert.strictEqual(data.limit, 200)
+    })
+
+    // `false` and `0` are the values a truthiness check silently replaces, and
+    // the reason this tests presence rather than falsiness.
+    test('should not overwrite a supplied falsy value', () => {
+      const data = applyDefaultsFromSchema('defaultsSchema', {
+        page: 0,
+        enabled: false,
+      })
+      assert.strictEqual(data.page, 0)
+      assert.strictEqual(data.enabled, false)
+    })
+
+    test('should fill defaults into nullish data', () => {
+      assert.strictEqual(
+        applyDefaultsFromSchema('defaultsSchema', undefined).page,
+        1
+      )
+      assert.strictEqual(
+        applyDefaultsFromSchema('defaultsSchema', null).limit,
+        50
+      )
+    })
+
+    test('should leave properties without a default absent', () => {
+      const data = applyDefaultsFromSchema('defaultsSchema', {})
+      assert.ok(!('name' in data))
+    })
+
+    // Two requests sharing one array default would let the first request's
+    // pushes show up in the second.
+    test('should clone object and array defaults per call', () => {
+      const first = applyDefaultsFromSchema('defaultsSchema', {})
+      const second = applyDefaultsFromSchema('defaultsSchema', {})
+      first.addons.push('leaked')
+      assert.deepStrictEqual(second.addons, [])
+    })
+
+    test('should return the same data when the schema has no defaults', () => {
+      const data = { name: 'x' }
+      assert.strictEqual(
+        applyDefaultsFromSchema('noDefaultsSchema', data),
+        data
+      )
+    })
+
+    test('should return data untouched for an unknown schema', () => {
+      const data = { name: 'x' }
+      assert.strictEqual(applyDefaultsFromSchema('nonExistent', data), data)
+    })
+
+    test('should leave a primitive body for the validator to reject', () => {
+      assert.strictEqual(
+        applyDefaultsFromSchema('defaultsSchema', 'nope'),
+        'nope'
+      )
+    })
+
+    test('should skip boolean schema properties', () => {
+      const data = applyDefaultsFromSchema('booleanPropSchema', {})
+      assert.strictEqual(data.page, 1)
+      assert.ok(!('isActive' in data))
+    })
+  })
+
   describe('validateSchema', () => {
     beforeEach(() => {
       resetPikkuState()

package/src/schema.ts

@@ -94,6 +94,57 @@ const validateAllSchemasLoaded = (
   }
 }
 
+/**
+ * Fill in absent top-level properties from their schema `default`.
+ *
+ * A `default` reaches the generated JSON Schema and keeps the property out of
+ * `required`, so omitting it validates — but nothing was ever filling it in.
+ * JSON Schema validators are pure by specification and none of the ones Pikku
+ * ships with (`@cfworker/json-schema`, and Ajv unless `useDefaults` is set)
+ * annotate the instance, so the function received `undefined` for a property
+ * its generated type declares as present. That is the worst shape a mismatch
+ * can take: validation permits the omission, the type says the value is there,
+ * and the body reads `undefined`.
+ *
+ * Applied unconditionally rather than alongside `coerceTopLevelDataFromSchema`,
+ * whose `coerceDataFromSchema` flag is about decoding transport-encoded values
+ * (a query string's `"1,2"` into an array). Defaults are a property of the
+ * schema, not of how the call arrived, so gating them on that flag would apply
+ * them over HTTP and skip them on a direct RPC invocation.
+ *
+ * Returns the data to use, which is a new object only when defaults had to be
+ * added to a nullish input — a call made with no arguments at all still gets
+ * them. Values are cloned so an object or array default (`[]`, `{}`) is never
+ * shared as one mutable instance across every request.
+ */
+export const applyDefaultsFromSchema = (
+  schemaName: string,
+  data: any,
+  packageName: string | null = null
+) => {
+  const schema = pikkuState(packageName, 'misc', 'schemas').get(schemaName)
+  if (!schema?.properties) return data
+
+  // A primitive body cannot carry named properties; leave it for the validator
+  // to reject rather than reshaping it into something that would pass.
+  if (data != null && typeof data !== 'object') return data
+
+  let result = data
+  for (const key in schema.properties) {
+    const property = schema.properties[key]
+    if (typeof property === 'boolean' || !('default' in property)) {
+      continue
+    }
+    // Allocated only once a default is actually found, so a schema without any
+    // leaves the caller's data (and its absence) exactly as it was.
+    result ??= {}
+    if (result[key] === undefined) {
+      result[key] = structuredClone(property.default)
+    }
+  }
+  return result
+}
+
 export const coerceTopLevelDataFromSchema = (
   schemaName: string,
   data: any,

package/src/services/in-memory-queue-service.test.ts

@@ -23,7 +23,10 @@ const registerWorker = (
 ) => {
   const calls = { count: 0 }
   const funcId = `queue_${queueName}`
-  pikkuState(null, 'queue', 'meta')[queueName] = { pikkuFuncId: funcId, name: queueName }
+  pikkuState(null, 'queue', 'meta')[queueName] = {
+    pikkuFuncId: funcId,
+    name: queueName,
+  }
   pikkuState(null, 'function', 'meta')[funcId] = {
     pikkuFuncId: funcId,
     inputSchemaName: null,
@@ -55,6 +58,68 @@ const waitUntil = async (
   return predicate()
 }
 
+// Register a queue worker that records the payload each delivery carried.
+const registerRecordingWorker = (queueName: string) => {
+  const received: any[] = []
+  const funcId = `queue_${queueName}`
+  pikkuState(null, 'queue', 'meta')[queueName] = {
+    pikkuFuncId: funcId,
+    name: queueName,
+  }
+  pikkuState(null, 'function', 'meta')[funcId] = {
+    pikkuFuncId: funcId,
+    inputSchemaName: null,
+    outputSchemaName: null,
+    sessionless: true,
+  } as any
+  wireQueueWorker({
+    name: queueName,
+    func: {
+      auth: false,
+      func: async (_services: any, data: any) => {
+        received.push(data)
+      },
+    },
+  } as any)
+  return received
+}
+
+describe('InMemoryQueueService payload isolation', () => {
+  test('a job carries the payload as it was when added', async () => {
+    const received = registerRecordingWorker('mutated')
+    const queue = new InMemoryQueueService()
+    const payload = { items: ['a'] }
+
+    await queue.add('mutated', payload, { delay: 0 })
+    payload.items.push('b')
+
+    assert.equal(await waitUntil(() => received.length === 1), true)
+    assert.deepEqual(
+      received[0].items,
+      ['a'],
+      'the worker saw a mutation made after the job was queued — a real queue would have serialised it away'
+    )
+  })
+
+  test('a payload arrives shaped the way a real queue would deliver it', async () => {
+    const received = registerRecordingWorker('serialised')
+    const queue = new InMemoryQueueService()
+
+    await queue.add(
+      'serialised',
+      { when: new Date('2020-01-01T00:00:00.000Z') },
+      { delay: 0 }
+    )
+
+    assert.equal(await waitUntil(() => received.length === 1), true)
+    assert.equal(
+      received[0].when,
+      '2020-01-01T00:00:00.000Z',
+      'a Date survived as a Date, so dev behaviour diverges from every real queue backend'
+    )
+  })
+})
+
 describe('InMemoryQueueService retry', () => {
   test('redelivers a transiently-failing job until it succeeds', async () => {
     const calls = registerWorker('flaky', (count) => {

package/src/services/in-memory-queue-service.ts

@@ -11,6 +11,12 @@ import { runQueueJob } from '../wirings/queue/queue-runner.js'
  * a real queue — and redelivers a failed job up to `options.attempts` times with
  * backoff, so a transiently-failing workflow step recovers exactly as it would
  * on pg-boss/bullmq instead of being silently dropped on its first error.
+ *
+ * Payloads are JSON round-tripped on the way in, because every real backend
+ * puts the job on a wire (SQS body, Redis value, jsonb column) and the worker
+ * therefore never sees the caller's live object. Doing it here keeps dev
+ * behaviour honest, and keeps the callers — who cannot know which backend they
+ * are talking to — from having to serialise defensively.
  */
 export class InMemoryQueueService implements QueueService {
   readonly supportsResults = false
@@ -25,13 +31,15 @@ export class InMemoryQueueService implements QueueService {
     const maxAttempts = Math.max(1, options?.attempts ?? 1)
     let attemptsMade = 0
     const createdAt = new Date()
+    const payload: T =
+      data === undefined ? data : (JSON.parse(JSON.stringify(data)) as T)
 
     const runAttempt = async () => {
       attemptsMade++
       const job: QueueJob<T> = {
         id: jobId,
         queueName,
-        data,
+        data: payload,
         status: () => 'active',
         metadata: () => ({ attemptsMade, maxAttempts, createdAt }),
         pikkuUserId: options?.pikkuUserId,
@@ -41,7 +49,10 @@ export class InMemoryQueueService implements QueueService {
       } catch (e: any) {
         if (attemptsMade < maxAttempts) {
           // Transient failure — redeliver with backoff, mirroring a real queue.
-          setTimeout(runAttempt, this.backoffDelay(options?.backoff, attemptsMade))
+          setTimeout(
+            runAttempt,
+            this.backoffDelay(options?.backoff, attemptsMade)
+          )
         } else {
           console.error(
             `[InMemoryQueue] Job ${jobId} on ${queueName} failed after ${attemptsMade} attempt(s):`,