@pikku/core 0.12.74 → 0.12.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +397 -0
- package/dist/column-form.d.ts +32 -0
- package/dist/column-form.js +42 -0
- package/dist/crypto-utils.d.ts +15 -4
- package/dist/crypto-utils.js +18 -2
- package/dist/data-classification.d.ts +44 -0
- package/dist/function/functions.types.d.ts +23 -10
- package/dist/function/index.d.ts +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.js +3 -1
- package/dist/middleware/auth-bearer.js +2 -1
- package/dist/middleware/remote-auth.js +1 -1
- package/dist/remote.js +1 -1
- package/dist/secret-value.d.ts +56 -0
- package/dist/secret-value.js +46 -0
- package/dist/services/audit-service.d.ts +74 -4
- package/dist/services/audit-service.js +7 -5
- package/dist/services/credential-wire-service.d.ts +5 -0
- package/dist/services/credential-wire-service.js +9 -1
- package/dist/services/email-service.d.ts +2 -1
- package/dist/services/in-memory-workflow-service.d.ts +1 -0
- package/dist/services/in-memory-workflow-service.js +20 -0
- package/dist/services/index.d.ts +3 -3
- package/dist/services/index.js +1 -1
- package/dist/services/local-content-request-handler.d.ts +29 -0
- package/dist/services/local-content-request-handler.js +176 -0
- package/dist/services/local-secrets.d.ts +4 -3
- package/dist/services/local-secrets.js +7 -3
- package/dist/services/logger.d.ts +22 -5
- package/dist/services/queue-webhook-service.js +1 -1
- package/dist/services/scoped-secret-service.d.ts +4 -3
- package/dist/services/secret-service.d.ts +8 -3
- package/dist/services/typed-secret-service.d.ts +5 -4
- package/dist/services/webhook-service.d.ts +2 -1
- package/dist/testing/service-tests.js +6 -6
- package/dist/types/core.types.d.ts +25 -4
- package/dist/wirings/ai-agent/ai-agent-agui.js +13 -1
- package/dist/wirings/ai-agent/ai-agent-prepare.js +7 -1
- package/dist/wirings/ai-agent/ai-agent-runner.js +14 -2
- package/dist/wirings/ai-agent/ai-agent-stream.js +27 -1
- package/dist/wirings/ai-agent/ai-agent.types.d.ts +40 -0
- package/dist/wirings/ai-agent/index.d.ts +1 -1
- package/dist/wirings/ai-agent/index.js +1 -1
- package/dist/wirings/ai-agent/voice-input.d.ts +20 -0
- package/dist/wirings/ai-agent/voice-input.js +44 -9
- package/dist/wirings/ai-agent/voice-output.d.ts +15 -0
- package/dist/wirings/ai-agent/voice-output.js +10 -1
- package/dist/wirings/cli/channel/cli-raw-client-runner.d.ts +21 -3
- package/dist/wirings/cli/channel/cli-raw-client-runner.js +13 -5
- package/dist/wirings/cli/channel/index.d.ts +1 -0
- package/dist/wirings/persona/define-personas.d.ts +4 -0
- package/dist/wirings/persona/define-personas.js +4 -0
- package/dist/wirings/persona/persona.types.d.ts +11 -0
- package/dist/wirings/queue/queue-identity.js +2 -1
- package/dist/wirings/queue/queue.types.d.ts +2 -1
- package/dist/wirings/queue/signed-queue-service.d.ts +2 -1
- package/dist/wirings/rpc/remote-addon-auth.d.ts +2 -1
- package/dist/wirings/rpc/remote-addon-auth.js +6 -2
- package/dist/wirings/virtual-user/index.d.ts +3 -0
- package/dist/wirings/virtual-user/index.js +2 -0
- package/dist/wirings/virtual-user/prepare-virtual-user-run.d.ts +54 -0
- package/dist/wirings/virtual-user/prepare-virtual-user-run.js +49 -0
- package/dist/wirings/virtual-user/virtual-user-run-store.d.ts +90 -0
- package/dist/wirings/virtual-user/virtual-user-run-store.js +1 -0
- package/dist/wirings/workflow/dsl/workflow-dsl.types.d.ts +14 -10
- package/dist/wirings/workflow/pikku-scenario-service.js +1 -2
- package/dist/wirings/workflow/pikku-workflow-service.d.ts +35 -0
- package/dist/wirings/workflow/pikku-workflow-service.js +51 -0
- package/dist/wirings/workflow/scenario-prose.js +1 -1
- package/dist/wirings/workflow/scenario-step.types.d.ts +13 -7
- package/dist/wirings/workflow/workflow.types.d.ts +7 -0
- package/knowledge/decisions/internals/a-virtual-user-run-is-not-a-workflow-and-not-a-queued-job.md +48 -0
- package/knowledge/decisions/internals/core-column-form-is-an-axis-of-its-own.md +84 -0
- package/knowledge/decisions/internals/core-data-classification-brand-is-an-optional-property.md +9 -2
- package/knowledge/decisions/internals/index.md +4 -0
- package/knowledge/decisions/internals/one-project-shape-check-two-validators.md +53 -0
- package/knowledge/decisions/internals/scenarios-live-in-files-named-for-them.md +48 -0
- package/knowledge/decisions/internals/validate-checks-personas-through-a-shared-module.md +43 -0
- package/package.json +3 -2
- package/src/column-form.test.ts +97 -0
- package/src/column-form.ts +58 -0
- package/src/crypto-utils.ts +25 -6
- package/src/data-classification.ts +44 -0
- package/src/function/functions.types.ts +47 -10
- package/src/function/index.ts +1 -0
- package/src/index.ts +24 -2
- package/src/middleware/auth-bearer.test.ts +3 -2
- package/src/middleware/auth-bearer.ts +2 -1
- package/src/middleware/remote-auth.test.ts +2 -1
- package/src/middleware/remote-auth.ts +1 -1
- package/src/remote.test.ts +2 -1
- package/src/remote.ts +1 -1
- package/src/secret-value.test.ts +204 -0
- package/src/secret-value.ts +111 -0
- package/src/services/audit-service.ts +87 -9
- package/src/services/credential-wire-service.ts +9 -1
- package/src/services/email-service.ts +3 -1
- package/src/services/in-memory-workflow-service.ts +29 -0
- package/src/services/index.ts +3 -3
- package/src/services/local-content-request-handler.test.ts +202 -0
- package/src/services/local-content-request-handler.ts +267 -0
- package/src/services/local-secrets.test.ts +20 -5
- package/src/services/local-secrets.ts +15 -7
- package/src/services/logger.ts +27 -7
- package/src/services/queue-webhook-service.test.ts +2 -1
- package/src/services/queue-webhook-service.ts +1 -1
- package/src/services/scoped-secret-service.ts +4 -3
- package/src/services/secret-service.ts +8 -3
- package/src/services/typed-secret-service.ts +11 -7
- package/src/services/webhook-service.ts +4 -1
- package/src/testing/service-tests.ts +6 -6
- package/src/types/core.types.ts +25 -4
- package/src/wirings/ai-agent/ai-agent-agui.test.ts +16 -0
- package/src/wirings/ai-agent/ai-agent-agui.ts +14 -1
- package/src/wirings/ai-agent/ai-agent-prepare.ts +7 -1
- package/src/wirings/ai-agent/ai-agent-runner.ts +18 -2
- package/src/wirings/ai-agent/ai-agent-stream.ts +32 -1
- package/src/wirings/ai-agent/ai-agent.types.ts +45 -1
- package/src/wirings/ai-agent/index.ts +2 -0
- package/src/wirings/ai-agent/voice-input.test.ts +65 -0
- package/src/wirings/ai-agent/voice-input.ts +48 -9
- package/src/wirings/ai-agent/voice-output.test.ts +91 -1
- package/src/wirings/ai-agent/voice-output.ts +28 -1
- package/src/wirings/cli/channel/cli-raw-client-runner.ts +39 -9
- package/src/wirings/cli/channel/index.ts +4 -0
- package/src/wirings/persona/define-personas.ts +4 -0
- package/src/wirings/persona/persona.types.ts +11 -0
- package/src/wirings/queue/queue-identity.test.ts +2 -1
- package/src/wirings/queue/queue-identity.ts +4 -1
- package/src/wirings/queue/queue.types.ts +6 -1
- package/src/wirings/queue/signed-queue-service.ts +2 -1
- package/src/wirings/rpc/remote-addon-auth.ts +8 -3
- package/src/wirings/rpc/rpc-runner.test.ts +6 -4
- package/src/wirings/virtual-user/index.ts +12 -0
- package/src/wirings/virtual-user/prepare-virtual-user-run.test.ts +115 -0
- package/src/wirings/virtual-user/prepare-virtual-user-run.ts +95 -0
- package/src/wirings/virtual-user/virtual-user-run-store.ts +98 -0
- package/src/wirings/workflow/dsl/workflow-dsl.types.ts +14 -16
- package/src/wirings/workflow/pikku-scenario-service.ts +1 -10
- package/src/wirings/workflow/pikku-workflow-service.ts +67 -0
- package/src/wirings/workflow/scenario-prose.test.ts +5 -7
- package/src/wirings/workflow/scenario-prose.ts +1 -1
- package/src/wirings/workflow/scenario-service.test.ts +0 -1
- package/src/wirings/workflow/scenario-step.test.ts +4 -5
- package/src/wirings/workflow/scenario-step.types.ts +13 -7
- package/src/wirings/workflow/scenario-surface.test.ts +6 -5
- package/src/wirings/workflow/workflow-stalled-recovery.test.ts +106 -0
- package/src/wirings/workflow/workflow.types.ts +7 -0
- package/tsconfig.tsbuildinfo +1 -1
- package/tsconfig.type-tests.json +12 -0
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import { describe, test, before, after } from 'node:test'
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
import {
|
|
4
|
+
mkdtempSync,
|
|
5
|
+
mkdirSync,
|
|
6
|
+
writeFileSync,
|
|
7
|
+
readFileSync,
|
|
8
|
+
existsSync,
|
|
9
|
+
rmSync,
|
|
10
|
+
} from 'node:fs'
|
|
11
|
+
import { tmpdir } from 'node:os'
|
|
12
|
+
import { join } from 'node:path'
|
|
13
|
+
import type { JWTService, Logger } from './index.js'
|
|
14
|
+
import { LocalContent } from './local-content.js'
|
|
15
|
+
import { createLocalContentRequestHandler } from './local-content-request-handler.js'
|
|
16
|
+
|
|
17
|
+
const noopLogger = {
|
|
18
|
+
info: () => {},
|
|
19
|
+
error: () => {},
|
|
20
|
+
warn: () => {},
|
|
21
|
+
debug: () => {},
|
|
22
|
+
trace: () => {},
|
|
23
|
+
setLevel: () => {},
|
|
24
|
+
} as unknown as Logger
|
|
25
|
+
|
|
26
|
+
const fakeJWT = {
|
|
27
|
+
encode: async (_expiry: string, payload: unknown) =>
|
|
28
|
+
Buffer.from(JSON.stringify(payload)).toString('base64url'),
|
|
29
|
+
decode: async (token: string) =>
|
|
30
|
+
JSON.parse(Buffer.from(token, 'base64url').toString()),
|
|
31
|
+
} as unknown as JWTService
|
|
32
|
+
|
|
33
|
+
describe('createLocalContentRequestHandler', () => {
|
|
34
|
+
let tmpDir: string
|
|
35
|
+
let handler: ReturnType<typeof createLocalContentRequestHandler>
|
|
36
|
+
let content: LocalContent
|
|
37
|
+
|
|
38
|
+
before(() => {
|
|
39
|
+
tmpDir = mkdtempSync(join(tmpdir(), 'pikku-content-handler-'))
|
|
40
|
+
mkdirSync(join(tmpDir, 'bucket'), { recursive: true })
|
|
41
|
+
writeFileSync(join(tmpDir, 'bucket', 'existing.bin'), 'already here')
|
|
42
|
+
|
|
43
|
+
const config = {
|
|
44
|
+
localFileUploadPath: tmpDir,
|
|
45
|
+
uploadUrlPrefix: '/upload',
|
|
46
|
+
assetUrlPrefix: '/assets',
|
|
47
|
+
sizeLimit: '1mb',
|
|
48
|
+
}
|
|
49
|
+
content = new LocalContent(config, noopLogger, fakeJWT)
|
|
50
|
+
handler = createLocalContentRequestHandler({
|
|
51
|
+
content: config,
|
|
52
|
+
logger: noopLogger,
|
|
53
|
+
getJWT: () => fakeJWT,
|
|
54
|
+
})
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
after(() => {
|
|
58
|
+
rmSync(tmpDir, { recursive: true, force: true })
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
test('returns null for a path outside both prefixes', async () => {
|
|
62
|
+
const result = await handler(
|
|
63
|
+
new Request('http://localhost/uploadsomething')
|
|
64
|
+
)
|
|
65
|
+
assert.equal(result, null)
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
test('returns null for a method neither prefix handles', async () => {
|
|
69
|
+
const result = await handler(
|
|
70
|
+
new Request('http://localhost/upload/bucket/x.bin', { method: 'GET' })
|
|
71
|
+
)
|
|
72
|
+
assert.equal(result, null)
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
// Encoded SEPARATORS, not encoded dots. `%2e%2e` never reaches the handler:
|
|
76
|
+
// the URL parser treats it as a double-dot segment and resolves it away, so
|
|
77
|
+
// the path arrives already outside the prefix. `%2f` survives parsing intact
|
|
78
|
+
// and only becomes a separator when the handler decodes it — which is the one
|
|
79
|
+
// way a `..` can still be sitting in the key by the time it is resolved, and
|
|
80
|
+
// therefore the case the guard actually has to catch.
|
|
81
|
+
test('refuses an upload whose key escapes the content directory', async () => {
|
|
82
|
+
writeFileSync(join(tmpDir, '..', 'pikku-handler-secret.txt'), 'secret')
|
|
83
|
+
try {
|
|
84
|
+
const response = await handler(
|
|
85
|
+
new Request(
|
|
86
|
+
'http://localhost/upload/bucket/..%2f..%2fpikku-handler-secret.txt',
|
|
87
|
+
{ method: 'PUT', body: 'overwritten' }
|
|
88
|
+
)
|
|
89
|
+
)
|
|
90
|
+
assert.equal(response?.status, 400)
|
|
91
|
+
assert.equal(
|
|
92
|
+
readFileSync(join(tmpDir, '..', 'pikku-handler-secret.txt'), 'utf8'),
|
|
93
|
+
'secret'
|
|
94
|
+
)
|
|
95
|
+
} finally {
|
|
96
|
+
rmSync(join(tmpDir, '..', 'pikku-handler-secret.txt'), { force: true })
|
|
97
|
+
}
|
|
98
|
+
})
|
|
99
|
+
|
|
100
|
+
test('refuses an asset read whose key escapes the content directory', async () => {
|
|
101
|
+
const response = await handler(
|
|
102
|
+
new Request('http://localhost/assets/bucket/..%2f..%2fanything.txt')
|
|
103
|
+
)
|
|
104
|
+
assert.equal(response?.status, 400)
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
test('round-trips an upload and a signed read', async () => {
|
|
108
|
+
const { uploadUrl } = await content.getUploadURL({
|
|
109
|
+
bucket: 'bucket',
|
|
110
|
+
fileKey: 'round/trip.bin',
|
|
111
|
+
contentType: 'application/octet-stream',
|
|
112
|
+
} as any)
|
|
113
|
+
const put = await handler(
|
|
114
|
+
new Request(`http://localhost${uploadUrl}`, {
|
|
115
|
+
method: 'PUT',
|
|
116
|
+
body: 'payload',
|
|
117
|
+
})
|
|
118
|
+
)
|
|
119
|
+
assert.equal(put?.status, 200)
|
|
120
|
+
|
|
121
|
+
const signed = await content.signContentKey({
|
|
122
|
+
bucket: 'bucket',
|
|
123
|
+
contentKey: 'round/trip.bin',
|
|
124
|
+
dateLessThan: new Date(Date.now() + 60_000),
|
|
125
|
+
} as any)
|
|
126
|
+
const get = await handler(new Request(`http://localhost${signed}`))
|
|
127
|
+
assert.equal(get?.status, 200)
|
|
128
|
+
assert.equal(await get!.text(), 'payload')
|
|
129
|
+
})
|
|
130
|
+
|
|
131
|
+
test('HEAD returns the length without the body', async () => {
|
|
132
|
+
const signed = await content.signContentKey({
|
|
133
|
+
bucket: 'bucket',
|
|
134
|
+
contentKey: 'existing.bin',
|
|
135
|
+
dateLessThan: new Date(Date.now() + 60_000),
|
|
136
|
+
} as any)
|
|
137
|
+
const response = await handler(
|
|
138
|
+
new Request(`http://localhost${signed}`, { method: 'HEAD' })
|
|
139
|
+
)
|
|
140
|
+
assert.equal(response?.status, 200)
|
|
141
|
+
assert.equal(
|
|
142
|
+
response?.headers.get('content-length'),
|
|
143
|
+
String('already here'.length)
|
|
144
|
+
)
|
|
145
|
+
assert.equal(await response!.text(), '')
|
|
146
|
+
})
|
|
147
|
+
|
|
148
|
+
test('refuses a read with no signature at all', async () => {
|
|
149
|
+
const response = await handler(
|
|
150
|
+
new Request('http://localhost/assets/bucket/existing.bin')
|
|
151
|
+
)
|
|
152
|
+
assert.equal(response?.status, 403)
|
|
153
|
+
})
|
|
154
|
+
|
|
155
|
+
test('refuses a signature that verifies but names another path', async () => {
|
|
156
|
+
const signed = await content.signContentKey({
|
|
157
|
+
bucket: 'bucket',
|
|
158
|
+
contentKey: 'existing.bin',
|
|
159
|
+
dateLessThan: new Date(Date.now() + 60_000),
|
|
160
|
+
} as any)
|
|
161
|
+
writeFileSync(join(tmpDir, 'bucket', 'sibling.bin'), 'sibling')
|
|
162
|
+
const query = signed.slice(signed.indexOf('?'))
|
|
163
|
+
const response = await handler(
|
|
164
|
+
new Request(`http://localhost/assets/bucket/sibling.bin${query}`)
|
|
165
|
+
)
|
|
166
|
+
assert.equal(response?.status, 403)
|
|
167
|
+
})
|
|
168
|
+
|
|
169
|
+
test('refuses signed reads when no JWT service is available', async () => {
|
|
170
|
+
const unverifiable = createLocalContentRequestHandler({
|
|
171
|
+
content: {
|
|
172
|
+
localFileUploadPath: tmpDir,
|
|
173
|
+
uploadUrlPrefix: '/upload',
|
|
174
|
+
assetUrlPrefix: '/assets',
|
|
175
|
+
},
|
|
176
|
+
logger: noopLogger,
|
|
177
|
+
getJWT: () => undefined,
|
|
178
|
+
})
|
|
179
|
+
const signed = await content.signContentKey({
|
|
180
|
+
bucket: 'bucket',
|
|
181
|
+
contentKey: 'existing.bin',
|
|
182
|
+
dateLessThan: new Date(Date.now() + 60_000),
|
|
183
|
+
} as any)
|
|
184
|
+
const response = await unverifiable(
|
|
185
|
+
new Request(`http://localhost${signed}`)
|
|
186
|
+
)
|
|
187
|
+
assert.equal(response?.status, 403)
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
test('rejects a body over the size limit without writing it', async () => {
|
|
191
|
+
const response = await handler(
|
|
192
|
+
new Request('http://localhost/upload/bucket/big.bin', {
|
|
193
|
+
method: 'PUT',
|
|
194
|
+
body: Buffer.alloc(1024 * 1024 + 1),
|
|
195
|
+
})
|
|
196
|
+
)
|
|
197
|
+
assert.equal(response?.status, 413)
|
|
198
|
+
// Abandoning the stream must leave nothing behind — a partial file here
|
|
199
|
+
// would be a truncated asset that later reads would serve as if whole.
|
|
200
|
+
assert.equal(existsSync(join(tmpDir, 'bucket', 'big.bin')), false)
|
|
201
|
+
})
|
|
202
|
+
})
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import { createReadStream } from 'fs'
|
|
2
|
+
import { mkdir, stat, writeFile } from 'fs/promises'
|
|
3
|
+
import { normalize, resolve } from 'path'
|
|
4
|
+
import { Readable } from 'stream'
|
|
5
|
+
import type { JWTService, Logger } from '@pikku/core/services'
|
|
6
|
+
import { signedContentPath, type LocalContentConfig } from './local-content.js'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The server half of {@link LocalContent}.
|
|
10
|
+
*
|
|
11
|
+
* `LocalContent` hands out `PUT <uploadUrlPrefix>/<key>` upload URLs and signed
|
|
12
|
+
* `GET <assetUrlPrefix>/<key>` read URLs, but it cannot answer either: it is a
|
|
13
|
+
* `ContentService`, not a transport. Something in the serving path has to, and
|
|
14
|
+
* until now only `@pikku/node-http-server` did — so the very same project served
|
|
15
|
+
* under Bun handed the browser upload URLs that 404ed, with nothing naming the
|
|
16
|
+
* cause.
|
|
17
|
+
*
|
|
18
|
+
* Expressed in Web `Request`/`Response` so every runtime can share one
|
|
19
|
+
* implementation rather than each re-deriving the signature check. Returns
|
|
20
|
+
* `null` for anything that is not a content request, which is the caller's
|
|
21
|
+
* signal to carry on with its normal routing.
|
|
22
|
+
*/
|
|
23
|
+
export type LocalContentRequestHandler = (
|
|
24
|
+
request: Request
|
|
25
|
+
) => Promise<Response | null>
|
|
26
|
+
|
|
27
|
+
export type LocalContentRequestHandlerOptions = {
|
|
28
|
+
content: LocalContentConfig
|
|
29
|
+
logger: Logger
|
|
30
|
+
/**
|
|
31
|
+
* Resolved per request rather than passed by value: a runtime may only be
|
|
32
|
+
* able to reach the signing service through `singletonServices`, which is not
|
|
33
|
+
* populated until after the server is constructed.
|
|
34
|
+
*/
|
|
35
|
+
getJWT: () => JWTService | undefined
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const matchesPrefix = (pathname: string, prefix: string) =>
|
|
39
|
+
pathname === prefix || pathname.startsWith(`${prefix}/`)
|
|
40
|
+
|
|
41
|
+
const contentKey = (pathname: string, prefix: string) =>
|
|
42
|
+
pathname.slice(prefix.length).replace(/^\/+/, '')
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Resolve a key against the content root, or `null` if it escapes. `normalize`
|
|
46
|
+
* first so `..` segments are collapsed before the prefix check, and the
|
|
47
|
+
* comparison carries a trailing separator so a sibling directory whose name
|
|
48
|
+
* merely starts with the root's cannot pass as being inside it.
|
|
49
|
+
*/
|
|
50
|
+
const toTargetPath = (basePath: string, key: string): string | null => {
|
|
51
|
+
const normalizedBasePath = resolve(basePath)
|
|
52
|
+
const targetPath = resolve(normalizedBasePath, normalize(key))
|
|
53
|
+
return targetPath.startsWith(`${normalizedBasePath}/`) ? targetPath : null
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const parseSizeLimit = (sizeLimit: string): number => {
|
|
57
|
+
const match = /^(\d+(?:\.\d+)?)(b|kb|mb|gb)?$/i.exec(sizeLimit.trim())
|
|
58
|
+
if (!match) {
|
|
59
|
+
throw new Error(`Invalid size limit: ${sizeLimit}`)
|
|
60
|
+
}
|
|
61
|
+
const value = Number(match[1])
|
|
62
|
+
const unit = (match[2] ?? 'b').toLowerCase()
|
|
63
|
+
const multiplier =
|
|
64
|
+
unit === 'gb'
|
|
65
|
+
? 1024 * 1024 * 1024
|
|
66
|
+
: unit === 'mb'
|
|
67
|
+
? 1024 * 1024
|
|
68
|
+
: unit === 'kb'
|
|
69
|
+
? 1024
|
|
70
|
+
: 1
|
|
71
|
+
return value * multiplier
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const text = (status: number, body: string) =>
|
|
75
|
+
new Response(body, {
|
|
76
|
+
status,
|
|
77
|
+
headers: { 'content-type': 'text/plain; charset=utf-8' },
|
|
78
|
+
})
|
|
79
|
+
|
|
80
|
+
export const createLocalContentRequestHandler = ({
|
|
81
|
+
content,
|
|
82
|
+
logger,
|
|
83
|
+
getJWT,
|
|
84
|
+
}: LocalContentRequestHandlerOptions): LocalContentRequestHandler => {
|
|
85
|
+
// Logged at most once. An unverifiable request is attacker-triggerable, so
|
|
86
|
+
// this reports a startup misconfiguration rather than per-request news.
|
|
87
|
+
let loggedMissingJWT = false
|
|
88
|
+
|
|
89
|
+
const validateSignedAssetRequest = async (
|
|
90
|
+
requestUrl: URL
|
|
91
|
+
): Promise<{ ok: true } | { ok: false; status: number; body: string }> => {
|
|
92
|
+
const signedAtValue = requestUrl.searchParams.get('signedAt')
|
|
93
|
+
const expiresAtValue = requestUrl.searchParams.get('expiresAt')
|
|
94
|
+
const notBeforeValue = requestUrl.searchParams.get('notBefore')
|
|
95
|
+
const signature = requestUrl.searchParams.get('signature')
|
|
96
|
+
|
|
97
|
+
if (!signedAtValue || !expiresAtValue) {
|
|
98
|
+
return { ok: false, status: 403, body: 'Signed URL required' }
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const signedAt = Number(signedAtValue)
|
|
102
|
+
const expiresAt = Number(expiresAtValue)
|
|
103
|
+
const notBefore =
|
|
104
|
+
notBeforeValue == null ? undefined : Number(notBeforeValue)
|
|
105
|
+
|
|
106
|
+
if (
|
|
107
|
+
!Number.isFinite(signedAt) ||
|
|
108
|
+
!Number.isFinite(expiresAt) ||
|
|
109
|
+
(notBefore != null && !Number.isFinite(notBefore))
|
|
110
|
+
) {
|
|
111
|
+
return { ok: false, status: 403, body: 'Invalid signed URL' }
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const now = Date.now()
|
|
115
|
+
if (now > expiresAt || (notBefore != null && now < notBefore)) {
|
|
116
|
+
return { ok: false, status: 403, body: 'Signed URL expired' }
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const jwt = getJWT()
|
|
120
|
+
if (!jwt) {
|
|
121
|
+
if (!loggedMissingJWT) {
|
|
122
|
+
loggedMissingJWT = true
|
|
123
|
+
logger.error(
|
|
124
|
+
'pikku: refusing signed asset reads — no JWTService is available to verify them. Pass `contentSigningJWT` (the same service LocalContent signs with) or expose it as `singletonServices.jwt`.'
|
|
125
|
+
)
|
|
126
|
+
}
|
|
127
|
+
return { ok: false, status: 403, body: 'Invalid signed URL' }
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
if (!signature) {
|
|
131
|
+
return { ok: false, status: 403, body: 'Signed URL signature required' }
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
try {
|
|
135
|
+
const payload = await jwt.decode<{
|
|
136
|
+
signedAt?: number
|
|
137
|
+
expiresAt?: number
|
|
138
|
+
notBefore?: number
|
|
139
|
+
path?: string
|
|
140
|
+
}>(signature)
|
|
141
|
+
|
|
142
|
+
// Every claim is compared, the path included: without it a signature
|
|
143
|
+
// minted for one asset would read any other.
|
|
144
|
+
if (
|
|
145
|
+
payload.signedAt !== signedAt ||
|
|
146
|
+
payload.expiresAt !== expiresAt ||
|
|
147
|
+
payload.notBefore !== notBefore ||
|
|
148
|
+
payload.path !== signedContentPath(requestUrl.pathname)
|
|
149
|
+
) {
|
|
150
|
+
return { ok: false, status: 403, body: 'Invalid signed URL' }
|
|
151
|
+
}
|
|
152
|
+
} catch {
|
|
153
|
+
return { ok: false, status: 403, body: 'Invalid signed URL' }
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
return { ok: true }
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const handleUpload = async (
|
|
160
|
+
request: Request,
|
|
161
|
+
pathname: string
|
|
162
|
+
): Promise<Response> => {
|
|
163
|
+
const key = contentKey(pathname, content.uploadUrlPrefix)
|
|
164
|
+
const targetPath = toTargetPath(content.localFileUploadPath, key)
|
|
165
|
+
if (!targetPath) {
|
|
166
|
+
return text(400, 'Invalid path')
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const maxBytes = parseSizeLimit(content.sizeLimit ?? '1mb')
|
|
170
|
+
|
|
171
|
+
// Counted as it arrives and abandoned the moment it goes over, so an
|
|
172
|
+
// oversized upload costs the limit rather than its own size — `arrayBuffer()`
|
|
173
|
+
// would have to hold all of it first, which hands an unauthenticated caller
|
|
174
|
+
// a way to spend the server's memory. Mirrors node-http-server's
|
|
175
|
+
// `readRequestBody`, which aborts the same way.
|
|
176
|
+
const chunks: Buffer[] = []
|
|
177
|
+
let bytesRead = 0
|
|
178
|
+
const reader = request.body?.getReader()
|
|
179
|
+
if (reader) {
|
|
180
|
+
try {
|
|
181
|
+
for (;;) {
|
|
182
|
+
const { done, value } = await reader.read()
|
|
183
|
+
if (done) break
|
|
184
|
+
bytesRead += value.byteLength
|
|
185
|
+
if (bytesRead > maxBytes) {
|
|
186
|
+
await reader.cancel()
|
|
187
|
+
return text(413, 'Content too large')
|
|
188
|
+
}
|
|
189
|
+
chunks.push(Buffer.from(value))
|
|
190
|
+
}
|
|
191
|
+
} finally {
|
|
192
|
+
reader.releaseLock()
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
await mkdir(resolve(targetPath, '..'), { recursive: true })
|
|
197
|
+
await writeFile(targetPath, Buffer.concat(chunks))
|
|
198
|
+
return new Response(null, { status: 200 })
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const handleAsset = async (
|
|
202
|
+
request: Request,
|
|
203
|
+
requestUrl: URL,
|
|
204
|
+
pathname: string
|
|
205
|
+
): Promise<Response> => {
|
|
206
|
+
const key = contentKey(pathname, content.assetUrlPrefix)
|
|
207
|
+
const targetPath = toTargetPath(content.localFileUploadPath, key)
|
|
208
|
+
if (!targetPath) {
|
|
209
|
+
return text(400, 'Invalid path')
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
const signed = await validateSignedAssetRequest(requestUrl)
|
|
213
|
+
if (!signed.ok) {
|
|
214
|
+
return text(signed.status, signed.body)
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
try {
|
|
218
|
+
const file = await stat(targetPath)
|
|
219
|
+
if (!file.isFile()) {
|
|
220
|
+
return new Response(null, { status: 404 })
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const headers = {
|
|
224
|
+
'content-length': String(file.size),
|
|
225
|
+
'content-type': 'application/octet-stream',
|
|
226
|
+
}
|
|
227
|
+
if (request.method === 'HEAD') {
|
|
228
|
+
return new Response(null, { status: 200, headers })
|
|
229
|
+
}
|
|
230
|
+
// Streamed rather than buffered: assets are user uploads, and their size
|
|
231
|
+
// is bounded by `sizeLimit` at write time, not by anything here.
|
|
232
|
+
return new Response(
|
|
233
|
+
Readable.toWeb(createReadStream(targetPath)) as ReadableStream,
|
|
234
|
+
{ status: 200, headers }
|
|
235
|
+
)
|
|
236
|
+
} catch {
|
|
237
|
+
return new Response(null, { status: 404 })
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
return async (request) => {
|
|
242
|
+
let requestUrl: URL
|
|
243
|
+
try {
|
|
244
|
+
requestUrl = new URL(request.url)
|
|
245
|
+
} catch {
|
|
246
|
+
return null
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const pathname = decodeURIComponent(requestUrl.pathname)
|
|
250
|
+
|
|
251
|
+
if (
|
|
252
|
+
request.method === 'PUT' &&
|
|
253
|
+
matchesPrefix(pathname, content.uploadUrlPrefix)
|
|
254
|
+
) {
|
|
255
|
+
return handleUpload(request, pathname)
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
if (
|
|
259
|
+
(request.method === 'GET' || request.method === 'HEAD') &&
|
|
260
|
+
matchesPrefix(pathname, content.assetUrlPrefix)
|
|
261
|
+
) {
|
|
262
|
+
return handleAsset(request, requestUrl, pathname)
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
return null
|
|
266
|
+
}
|
|
267
|
+
}
|
|
@@ -7,14 +7,14 @@ describe('LocalSecretService', () => {
|
|
|
7
7
|
test('should get secret from local storage', async () => {
|
|
8
8
|
const service = new LocalSecretService()
|
|
9
9
|
await service.setSecret('MY_KEY', { token: 'abc' })
|
|
10
|
-
const result = await service.getSecret('MY_KEY')
|
|
10
|
+
const result = (await service.getSecret('MY_KEY')).reveal()
|
|
11
11
|
assert.deepStrictEqual(result, { token: 'abc' })
|
|
12
12
|
})
|
|
13
13
|
|
|
14
14
|
test('should get secret from environment variables', async () => {
|
|
15
15
|
const vars = new LocalVariablesService({ MY_SECRET: 'secret-value' })
|
|
16
16
|
const service = new LocalSecretService(vars)
|
|
17
|
-
const result = await service.getSecret('MY_SECRET')
|
|
17
|
+
const result = (await service.getSecret('MY_SECRET')).reveal()
|
|
18
18
|
assert.strictEqual(result, 'secret-value')
|
|
19
19
|
})
|
|
20
20
|
|
|
@@ -29,14 +29,14 @@ describe('LocalSecretService', () => {
|
|
|
29
29
|
test('should get JSON secret from local storage', async () => {
|
|
30
30
|
const service = new LocalSecretService()
|
|
31
31
|
await service.setSecret('JSON_KEY', { data: 42 })
|
|
32
|
-
const result = await service.getSecret('JSON_KEY')
|
|
32
|
+
const result = (await service.getSecret('JSON_KEY')).reveal()
|
|
33
33
|
assert.deepStrictEqual(result, { data: 42 })
|
|
34
34
|
})
|
|
35
35
|
|
|
36
36
|
test('should get JSON secret from environment variables', async () => {
|
|
37
37
|
const vars = new LocalVariablesService({ CONFIG: '{"port":3000}' })
|
|
38
38
|
const service = new LocalSecretService(vars)
|
|
39
|
-
const result = await service.getSecret('CONFIG')
|
|
39
|
+
const result = (await service.getSecret('CONFIG')).reveal()
|
|
40
40
|
assert.deepStrictEqual(result, { port: 3000 })
|
|
41
41
|
})
|
|
42
42
|
|
|
@@ -52,7 +52,7 @@ describe('LocalSecretService', () => {
|
|
|
52
52
|
const vars = new LocalVariablesService({ KEY: 'env-value' })
|
|
53
53
|
const service = new LocalSecretService(vars)
|
|
54
54
|
await service.setSecret('KEY', 'local-value')
|
|
55
|
-
const result = await service.getSecret('KEY')
|
|
55
|
+
const result = (await service.getSecret('KEY')).reveal()
|
|
56
56
|
assert.strictEqual(result, 'local-value')
|
|
57
57
|
})
|
|
58
58
|
|
|
@@ -78,3 +78,18 @@ describe('LocalSecretService', () => {
|
|
|
78
78
|
assert.strictEqual(await service.hasSecret('DEL_KEY'), false)
|
|
79
79
|
})
|
|
80
80
|
})
|
|
81
|
+
|
|
82
|
+
describe('LocalSecretService round-trips a wrapped secret', () => {
|
|
83
|
+
test('setSecret unwraps rather than storing the redaction', async () => {
|
|
84
|
+
const service = new LocalSecretService()
|
|
85
|
+
await service.setSecret('SOURCE', 'sk-live-DEADBEEF')
|
|
86
|
+
const read = await service.getSecret('SOURCE')
|
|
87
|
+
|
|
88
|
+
await service.setSecret('COPY', read)
|
|
89
|
+
|
|
90
|
+
assert.strictEqual(
|
|
91
|
+
(await service.getSecret('COPY')).reveal(),
|
|
92
|
+
'sk-live-DEADBEEF'
|
|
93
|
+
)
|
|
94
|
+
})
|
|
95
|
+
})
|
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
import {
|
|
2
|
+
createSecretValue,
|
|
3
|
+
isSecretValue,
|
|
4
|
+
type SecretValue,
|
|
5
|
+
} from '../secret-value.js'
|
|
1
6
|
import { LocalVariablesService } from './local-variables.js'
|
|
2
|
-
import type { SecretService } from './secret-service.js'
|
|
7
|
+
import type { SecretService, SecretValues } from './secret-service.js'
|
|
3
8
|
import type { VariablesService } from './variables-service.js'
|
|
4
9
|
|
|
5
10
|
export class LocalSecretService implements SecretService {
|
|
@@ -17,23 +22,26 @@ export class LocalSecretService implements SecretService {
|
|
|
17
22
|
private variables: VariablesService = new LocalVariablesService()
|
|
18
23
|
) {}
|
|
19
24
|
|
|
20
|
-
public async getSecret<T = string>(key: string): Promise<T
|
|
25
|
+
public async getSecret<T = string>(key: string): Promise<SecretValue<T>> {
|
|
21
26
|
const localValue = this.localSecrets.get(key)
|
|
22
27
|
if (localValue) {
|
|
23
|
-
return this.parseSecret<T>(localValue)
|
|
28
|
+
return createSecretValue(this.parseSecret<T>(localValue))
|
|
24
29
|
}
|
|
25
30
|
|
|
26
31
|
const value = await this.variables.get(key)
|
|
27
32
|
if (value) {
|
|
28
|
-
return this.parseSecret<T>(value)
|
|
33
|
+
return createSecretValue(this.parseSecret<T>(value))
|
|
29
34
|
}
|
|
30
35
|
throw new Error('Requested secret not found')
|
|
31
36
|
}
|
|
32
37
|
|
|
33
38
|
public async setSecret(key: string, value: unknown): Promise<void> {
|
|
39
|
+
// Storing the wrapper would serialize it to '[secret]', so unwrap first —
|
|
40
|
+
// writing a secret back to the vault is exactly what this method is for.
|
|
41
|
+
const raw = isSecretValue(value) ? value.reveal() : value
|
|
34
42
|
this.localSecrets.set(
|
|
35
43
|
key,
|
|
36
|
-
typeof
|
|
44
|
+
typeof raw === 'string' ? raw : JSON.stringify(raw)
|
|
37
45
|
)
|
|
38
46
|
}
|
|
39
47
|
|
|
@@ -51,13 +59,13 @@ export class LocalSecretService implements SecretService {
|
|
|
51
59
|
|
|
52
60
|
public async getSecrets<
|
|
53
61
|
T extends Record<string, unknown> = Record<string, unknown>,
|
|
54
|
-
>(keys: (keyof T & string)[]): Promise<Partial<T
|
|
62
|
+
>(keys: (keyof T & string)[]): Promise<Partial<SecretValues<T>>> {
|
|
55
63
|
const results = await Promise.allSettled(keys.map((k) => this.getSecret(k)))
|
|
56
64
|
const out: Record<string, unknown> = {}
|
|
57
65
|
keys.forEach((key, i) => {
|
|
58
66
|
if (results[i].status === 'fulfilled')
|
|
59
67
|
out[key] = (results[i] as PromiseFulfilledResult<unknown>).value
|
|
60
68
|
})
|
|
61
|
-
return out as Partial<T
|
|
69
|
+
return out as Partial<SecretValues<T>>
|
|
62
70
|
}
|
|
63
71
|
}
|
package/src/services/logger.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { Safe } from '../secret-value.js'
|
|
2
|
+
|
|
1
3
|
export enum LogLevel {
|
|
2
4
|
'trace',
|
|
3
5
|
'debug',
|
|
@@ -7,19 +9,37 @@ export enum LogLevel {
|
|
|
7
9
|
'critical',
|
|
8
10
|
}
|
|
9
11
|
|
|
12
|
+
/**
|
|
13
|
+
* A log line is the easiest place to leak a vault secret, so every parameter is
|
|
14
|
+
* `Safe<>`-guarded: a `SecretValue` anywhere in the message or the metadata,
|
|
15
|
+
* however deeply nested, collapses to `never` and fails the build. Reveal it
|
|
16
|
+
* first if you genuinely mean to log it.
|
|
17
|
+
*/
|
|
10
18
|
export interface Logger {
|
|
11
|
-
info
|
|
19
|
+
info<M extends string | Record<string, any>, A extends unknown[]>(
|
|
20
|
+
messageOrObj: Safe<M>,
|
|
21
|
+
...meta: { [K in keyof A]: Safe<A[K]> }
|
|
22
|
+
): void
|
|
12
23
|
|
|
13
|
-
warn
|
|
24
|
+
warn<M extends string | Record<string, any>, A extends unknown[]>(
|
|
25
|
+
messageOrObj: Safe<M>,
|
|
26
|
+
...meta: { [K in keyof A]: Safe<A[K]> }
|
|
27
|
+
): void
|
|
14
28
|
|
|
15
|
-
error(
|
|
16
|
-
messageOrObj:
|
|
17
|
-
...meta:
|
|
29
|
+
error<M extends string | Record<string, any> | Error, A extends unknown[]>(
|
|
30
|
+
messageOrObj: Safe<M>,
|
|
31
|
+
...meta: { [K in keyof A]: Safe<A[K]> }
|
|
18
32
|
): void
|
|
19
33
|
|
|
20
|
-
debug
|
|
34
|
+
debug<A extends unknown[]>(
|
|
35
|
+
message: string,
|
|
36
|
+
...meta: { [K in keyof A]: Safe<A[K]> }
|
|
37
|
+
): void
|
|
21
38
|
|
|
22
|
-
trace
|
|
39
|
+
trace?<A extends unknown[]>(
|
|
40
|
+
message: string,
|
|
41
|
+
...meta: { [K in keyof A]: Safe<A[K]> }
|
|
42
|
+
): void
|
|
23
43
|
|
|
24
44
|
setLevel(level: LogLevel): void
|
|
25
45
|
|
|
@@ -14,6 +14,7 @@ import { InMemoryQueueService } from './in-memory-queue-service.js'
|
|
|
14
14
|
import { wireQueueWorker } from '../wirings/queue/queue-runner.js'
|
|
15
15
|
import { resetPikkuState, pikkuState } from '../pikku-state.js'
|
|
16
16
|
import type { JobOptions } from '../wirings/queue/queue.types.js'
|
|
17
|
+
import { createSecretValue } from '../secret-value.js'
|
|
17
18
|
|
|
18
19
|
const noopLogger = { error() {}, info() {}, warn() {}, debug() {} }
|
|
19
20
|
|
|
@@ -132,7 +133,7 @@ describe('QueueWebhookService.send', () => {
|
|
|
132
133
|
secrets: {
|
|
133
134
|
getSecret: async (key: string) => {
|
|
134
135
|
assert.equal(key, 'WEBHOOK_SIGNING_KEY')
|
|
135
|
-
return 'resolved-key'
|
|
136
|
+
return createSecretValue('resolved-key')
|
|
136
137
|
},
|
|
137
138
|
},
|
|
138
139
|
})
|
|
@@ -52,7 +52,7 @@ export class QueueWebhookService extends WebhookService {
|
|
|
52
52
|
let secret = input.secret
|
|
53
53
|
if (secret === undefined && webhookConfig?.secret) {
|
|
54
54
|
// Naive read: caching is the secret service's concern, not ours.
|
|
55
|
-
secret = await services.secrets.getSecret(webhookConfig.secret)
|
|
55
|
+
secret = (await services.secrets.getSecret(webhookConfig.secret)).reveal()
|
|
56
56
|
if (!secret) {
|
|
57
57
|
services.logger.error(
|
|
58
58
|
`Webhook signing secret '${webhookConfig.secret}' (config.webhook.secret) resolved to nothing — outgoing webhooks will be sent UNSIGNED.`
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { SecretValue } from '../secret-value.js'
|
|
2
|
+
import type { SecretService, SecretValues } from './secret-service.js'
|
|
2
3
|
|
|
3
4
|
export class ScopedSecretService implements SecretService {
|
|
4
5
|
constructor(
|
|
@@ -12,7 +13,7 @@ export class ScopedSecretService implements SecretService {
|
|
|
12
13
|
}
|
|
13
14
|
}
|
|
14
15
|
|
|
15
|
-
async getSecret<T = string>(key: string): Promise<T
|
|
16
|
+
async getSecret<T = string>(key: string): Promise<SecretValue<T>> {
|
|
16
17
|
this.assertAllowed(key)
|
|
17
18
|
return this.secrets.getSecret<T>(key)
|
|
18
19
|
}
|
|
@@ -32,7 +33,7 @@ export class ScopedSecretService implements SecretService {
|
|
|
32
33
|
|
|
33
34
|
async getSecrets<T extends Record<string, unknown> = Record<string, unknown>>(
|
|
34
35
|
keys: (keyof T & string)[]
|
|
35
|
-
): Promise<Partial<T
|
|
36
|
+
): Promise<Partial<SecretValues<T>>> {
|
|
36
37
|
keys.forEach((k) => this.assertAllowed(k))
|
|
37
38
|
return this.secrets.getSecrets<T>(keys)
|
|
38
39
|
}
|