@ossy/platform 3.11.1 → 3.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +6 -7
- package/README.md +7 -6
- package/package.json +6 -6
- package/src/geoip/ensure-geolite2-country.js +119 -0
- package/src/geoip/ensure-geolite2-country.spec.js +140 -0
- package/src/geoip/extract-mmdb-from-tar-gz.js +68 -0
- package/src/geoip/extract-mmdb-from-tar-gz.spec.js +20 -0
- package/src/geoip/geolite2-country-path.js +10 -0
- package/src/geoip/tar-fixture.js +31 -0
- package/src/proxy-internal.js +29 -5
- package/src/proxy-internal.spec.js +62 -0
- package/src/runtime.js +13 -2
- package/src/server.js +16 -0
- package/src/site-loader.js +1 -1
- package/src/tasks/task-service.js +15 -12
- package/src/test/flow-runner.js +25 -3
- package/src/test/flow-shard.js +66 -0
- package/src/test/flow-shard.spec.js +73 -0
package/Dockerfile
CHANGED
|
@@ -1,16 +1,15 @@
|
|
|
1
1
|
# Build context: ossy/packages/platform directory.
|
|
2
|
-
# From repo root: docker build -f
|
|
3
|
-
#
|
|
2
|
+
# From repo root: docker build -f packages/platform/Dockerfile packages/platform
|
|
3
|
+
# Pushed by the manual "Deploy platform runtime" workflow (not npm Publish).
|
|
4
4
|
FROM node:24-bookworm-slim
|
|
5
5
|
|
|
6
6
|
WORKDIR /app
|
|
7
7
|
|
|
8
8
|
COPY package.json ./
|
|
9
9
|
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
|
|
13
|
-
RUN npm install --omit=dev --no-audit --no-fund --loglevel=error
|
|
10
|
+
# Production deps from npm. --legacy-peer-deps: transitive peers on an older
|
|
11
|
+
# lockstep patch must not fail the image when ranges are otherwise valid.
|
|
12
|
+
RUN npm install --omit=dev --no-audit --no-fund --loglevel=error --legacy-peer-deps
|
|
14
13
|
|
|
15
14
|
COPY src ./src
|
|
16
15
|
COPY docker-healthcheck.js ./docker-healthcheck.js
|
|
@@ -29,7 +28,7 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
|
|
|
29
28
|
|
|
30
29
|
# Required env vars at runtime:
|
|
31
30
|
# OSSY_API_KEY — Ossy API JWT for CMS reads
|
|
32
|
-
# OSSY_API_URL — (optional) override API base, default https://
|
|
31
|
+
# OSSY_API_URL — (optional) override API base, default https://ossy.se/api/v0
|
|
33
32
|
# PORT — (optional) override listen port, default 3000
|
|
34
33
|
# OSSY_SERVICE_NAME — (optional) `/health` service label, default runtime
|
|
35
34
|
CMD ["node", "src/runtime.js"]
|
package/README.md
CHANGED
|
@@ -11,12 +11,13 @@ At startup `@ossy/platform`:
|
|
|
11
11
|
3. Registers and runs all **startup hooks** (`*.startup.js`) in order.
|
|
12
12
|
4. Connects all **integrations** (`*.integration.js`) by calling `connect({ env })`.
|
|
13
13
|
5. Registers all **tasks** (`*.task.js`) with `TaskService` and starts the cron scheduler.
|
|
14
|
-
6.
|
|
15
|
-
7.
|
|
16
|
-
8.
|
|
17
|
-
9.
|
|
18
|
-
10.
|
|
19
|
-
11.
|
|
14
|
+
6. When `MAXMIND_LICENSE_KEY` is set, downloads GeoLite2-Country to `GEOLITE2_COUNTRY_MMDB` (default `/tmp/GeoLite2-Country.mmdb`) so Host location projections can resolve country. Missing key or a failed download does not prevent startup (#804).
|
|
15
|
+
7. Registers all **schemas** (`*.schema.js`) with `registerSchema`.
|
|
16
|
+
8. Rebuilds all **aggregates** (`*.aggregate.js`) from the event store.
|
|
17
|
+
9. Registers all **actions** (`*.action.js`) with `ActionService`.
|
|
18
|
+
10. Mounts **MCP** at `POST /mcp` and serves `GET /capabilities.json`.
|
|
19
|
+
11. Starts an Express server that routes requests to pages (`*.page.jsx`) and API handlers (`*.api.js`).
|
|
20
|
+
12. Auto-mounts every action at `POST /actions` (`{ action, payload }`).
|
|
20
21
|
|
|
21
22
|
Page SSR fills the `app:content` slot (see `PlatformShell` and `resolve-app-slots` in `@ossy/app`). App chrome uses namespaced keys such as `app:header` mapped from `export const slots` in `*.layout.jsx`.
|
|
22
23
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ossy/platform",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.13.0",
|
|
4
4
|
"description": "Ossy application server runtime",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -46,16 +46,16 @@
|
|
|
46
46
|
"@aws-sdk/util-format-url": "^3.972.17",
|
|
47
47
|
"@modelcontextprotocol/sdk": "^1.12.1",
|
|
48
48
|
"@ossy/config": "^3.0.9",
|
|
49
|
-
"@ossy/event-store": "^3.
|
|
49
|
+
"@ossy/event-store": "^3.12.0",
|
|
50
50
|
"@ossy/locale": "^3.4.0",
|
|
51
51
|
"@ossy/manifest": "^3.9.0",
|
|
52
52
|
"@ossy/observability": "^3.0.9",
|
|
53
53
|
"@ossy/policies": "^3.0.9",
|
|
54
54
|
"@ossy/schema": "^3.8.0",
|
|
55
|
-
"@ossy/sdk": "^3.
|
|
55
|
+
"@ossy/sdk": "^3.12.0",
|
|
56
56
|
"@ossy/tokens": "^3.11.1",
|
|
57
|
-
"@ossy/users": "^3.
|
|
58
|
-
"@ossy/workspaces": "^3.
|
|
57
|
+
"@ossy/users": "^3.12.0",
|
|
58
|
+
"@ossy/workspaces": "^3.12.0",
|
|
59
59
|
"cookie-parser": "^1.4.7",
|
|
60
60
|
"dotenv": ">=16.0.0 <18.0.0",
|
|
61
61
|
"express": ">=5.0.0 <6.0.0",
|
|
@@ -76,5 +76,5 @@
|
|
|
76
76
|
"Dockerfile",
|
|
77
77
|
"docker-healthcheck.js"
|
|
78
78
|
],
|
|
79
|
-
"gitHead": "
|
|
79
|
+
"gitHead": "7a02d1e056b731aa36f72e3f8d6df941101dbd50"
|
|
80
80
|
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import fs from 'node:fs'
|
|
2
|
+
import path from 'node:path'
|
|
3
|
+
import { createLogger } from '@ossy/observability'
|
|
4
|
+
import { extractMmdbFromTarGz } from './extract-mmdb-from-tar-gz.js'
|
|
5
|
+
import {
|
|
6
|
+
GEOLITE2_COUNTRY_DOWNLOAD_TIMEOUT_MS,
|
|
7
|
+
GEOLITE2_COUNTRY_MMDB_PATH,
|
|
8
|
+
} from './geolite2-country-path.js'
|
|
9
|
+
|
|
10
|
+
const log = createLogger('@ossy/platform/geoip')
|
|
11
|
+
|
|
12
|
+
const LEGACY_DOWNLOAD_URL = 'https://download.maxmind.com/app/geoip_download'
|
|
13
|
+
const PERMALINK_DOWNLOAD_URL = 'https://download.maxmind.com/geoip/databases/GeoLite2-Country/download?suffix=tar.gz'
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* @param {NodeJS.ProcessEnv} [env]
|
|
17
|
+
* @returns {string}
|
|
18
|
+
*/
|
|
19
|
+
export function resolveGeoLite2CountryPath (env = process.env) {
|
|
20
|
+
const raw = env.GEOLITE2_COUNTRY_MMDB ?? env.OSSY_GEOLITE2_COUNTRY_MMDB ?? ''
|
|
21
|
+
const trimmed = String(raw).trim()
|
|
22
|
+
return trimmed || GEOLITE2_COUNTRY_MMDB_PATH
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @param {NodeJS.ProcessEnv} [env]
|
|
27
|
+
* @returns {string | null}
|
|
28
|
+
*/
|
|
29
|
+
export function resolveMaxMindLicenseKey (env = process.env) {
|
|
30
|
+
const raw = env.MAXMIND_LICENSE_KEY ?? env.OSSY_MAXMIND_LICENSE_KEY ?? ''
|
|
31
|
+
const key = String(raw).trim()
|
|
32
|
+
return key || null
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* @param {NodeJS.ProcessEnv} [env]
|
|
37
|
+
* @returns {string | null}
|
|
38
|
+
*/
|
|
39
|
+
export function resolveMaxMindAccountId (env = process.env) {
|
|
40
|
+
const raw = env.MAXMIND_ACCOUNT_ID ?? env.OSSY_MAXMIND_ACCOUNT_ID ?? ''
|
|
41
|
+
const id = String(raw).trim()
|
|
42
|
+
return id || null
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* @param {{ licenseKey: string, accountId?: string | null }} options
|
|
47
|
+
* @returns {{ url: string, headers: Record<string, string> }}
|
|
48
|
+
*/
|
|
49
|
+
export function maxMindDownloadRequest ({ licenseKey, accountId }) {
|
|
50
|
+
if (accountId) {
|
|
51
|
+
const basic = Buffer.from(`${accountId}:${licenseKey}`, 'utf8').toString('base64')
|
|
52
|
+
return {
|
|
53
|
+
url: PERMALINK_DOWNLOAD_URL,
|
|
54
|
+
headers: { Authorization: `Basic ${basic}` },
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
const url = `${LEGACY_DOWNLOAD_URL}?edition_id=GeoLite2-Country&license_key=${encodeURIComponent(licenseKey)}&suffix=tar.gz`
|
|
58
|
+
return { url, headers: {} }
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Download GeoLite2-Country when `MAXMIND_LICENSE_KEY` is set.
|
|
63
|
+
*
|
|
64
|
+
* Shared by every `@ossy/platform` `startServer` image (ossy.se, plexus-sanitas,
|
|
65
|
+
* future `services[]` sites). Missing key / failed download is non-fatal:
|
|
66
|
+
* the process starts and location tiles omit `countryCode` (#804).
|
|
67
|
+
*
|
|
68
|
+
* @param {{
|
|
69
|
+
* env?: NodeJS.ProcessEnv,
|
|
70
|
+
* fetchImpl?: typeof fetch,
|
|
71
|
+
* }} [options]
|
|
72
|
+
* @returns {Promise<string | null>} destination path when a readable `.mmdb` is in place
|
|
73
|
+
*/
|
|
74
|
+
export async function ensureGeoLite2Country (options = {}) {
|
|
75
|
+
const env = options.env ?? process.env
|
|
76
|
+
const fetchImpl = options.fetchImpl ?? fetch
|
|
77
|
+
const dest = resolveGeoLite2CountryPath(env)
|
|
78
|
+
|
|
79
|
+
if (fs.existsSync(dest)) {
|
|
80
|
+
env.GEOLITE2_COUNTRY_MMDB = dest
|
|
81
|
+
log.info(`GeoLite2-Country already present at ${dest}`)
|
|
82
|
+
return dest
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const licenseKey = resolveMaxMindLicenseKey(env)
|
|
86
|
+
if (!licenseKey) {
|
|
87
|
+
log.info('MAXMIND_LICENSE_KEY unset — skipping GeoLite2-Country download; location tiles will omit countryCode')
|
|
88
|
+
return null
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const accountId = resolveMaxMindAccountId(env)
|
|
92
|
+
const { url, headers } = maxMindDownloadRequest({ licenseKey, accountId })
|
|
93
|
+
|
|
94
|
+
try {
|
|
95
|
+
log.info('Downloading GeoLite2-Country from MaxMind')
|
|
96
|
+
const response = await fetchImpl(url, {
|
|
97
|
+
headers,
|
|
98
|
+
signal: AbortSignal.timeout(GEOLITE2_COUNTRY_DOWNLOAD_TIMEOUT_MS),
|
|
99
|
+
})
|
|
100
|
+
if (!response.ok) {
|
|
101
|
+
throw new Error(`MaxMind download HTTP ${response.status}`)
|
|
102
|
+
}
|
|
103
|
+
const gzipped = Buffer.from(await response.arrayBuffer())
|
|
104
|
+
const mmdb = extractMmdbFromTarGz(gzipped)
|
|
105
|
+
|
|
106
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true })
|
|
107
|
+
const staging = `${dest}.download`
|
|
108
|
+
fs.writeFileSync(staging, mmdb)
|
|
109
|
+
fs.renameSync(staging, dest)
|
|
110
|
+
env.GEOLITE2_COUNTRY_MMDB = dest
|
|
111
|
+
log.info(`Wrote GeoLite2-Country to ${dest}`)
|
|
112
|
+
return dest
|
|
113
|
+
} catch (err) {
|
|
114
|
+
log.warn('GeoLite2-Country download failed — location tiles will omit countryCode', {
|
|
115
|
+
error: err instanceof Error ? err.message : String(err),
|
|
116
|
+
})
|
|
117
|
+
return null
|
|
118
|
+
}
|
|
119
|
+
}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import fs from 'node:fs'
|
|
2
|
+
import os from 'node:os'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
import { afterEach, describe, expect, it } from '@jest/globals'
|
|
5
|
+
import {
|
|
6
|
+
ensureGeoLite2Country,
|
|
7
|
+
maxMindDownloadRequest,
|
|
8
|
+
resolveGeoLite2CountryPath,
|
|
9
|
+
resolveMaxMindLicenseKey,
|
|
10
|
+
} from './ensure-geolite2-country.js'
|
|
11
|
+
import { GEOLITE2_COUNTRY_MMDB_PATH } from './geolite2-country-path.js'
|
|
12
|
+
import { makeUstarTarGz } from './tar-fixture.js'
|
|
13
|
+
|
|
14
|
+
const originalGeo = process.env.GEOLITE2_COUNTRY_MMDB
|
|
15
|
+
const originalOssyGeo = process.env.OSSY_GEOLITE2_COUNTRY_MMDB
|
|
16
|
+
const originalKey = process.env.MAXMIND_LICENSE_KEY
|
|
17
|
+
const originalOssyKey = process.env.OSSY_MAXMIND_LICENSE_KEY
|
|
18
|
+
const originalAccount = process.env.MAXMIND_ACCOUNT_ID
|
|
19
|
+
|
|
20
|
+
function restoreEnv () {
|
|
21
|
+
restore('GEOLITE2_COUNTRY_MMDB', originalGeo)
|
|
22
|
+
restore('OSSY_GEOLITE2_COUNTRY_MMDB', originalOssyGeo)
|
|
23
|
+
restore('MAXMIND_LICENSE_KEY', originalKey)
|
|
24
|
+
restore('OSSY_MAXMIND_LICENSE_KEY', originalOssyKey)
|
|
25
|
+
restore('MAXMIND_ACCOUNT_ID', originalAccount)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function restore (key, previous) {
|
|
29
|
+
if (previous == null) delete process.env[key]
|
|
30
|
+
else process.env[key] = previous
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
describe('resolveGeoLite2CountryPath', () => {
|
|
34
|
+
afterEach(restoreEnv)
|
|
35
|
+
|
|
36
|
+
it('defaults to the well-known container path', () => {
|
|
37
|
+
expect(resolveGeoLite2CountryPath({})).toBe(GEOLITE2_COUNTRY_MMDB_PATH)
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
it('prefers GEOLITE2_COUNTRY_MMDB', () => {
|
|
41
|
+
expect(resolveGeoLite2CountryPath({ GEOLITE2_COUNTRY_MMDB: '/var/lib/GeoLite2-Country.mmdb' }))
|
|
42
|
+
.toBe('/var/lib/GeoLite2-Country.mmdb')
|
|
43
|
+
})
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
describe('resolveMaxMindLicenseKey', () => {
|
|
47
|
+
it('returns null when unset', () => {
|
|
48
|
+
expect(resolveMaxMindLicenseKey({})).toBeNull()
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
it('trims OSSY_MAXMIND_LICENSE_KEY', () => {
|
|
52
|
+
expect(resolveMaxMindLicenseKey({ OSSY_MAXMIND_LICENSE_KEY: ' abc ' })).toBe('abc')
|
|
53
|
+
})
|
|
54
|
+
})
|
|
55
|
+
|
|
56
|
+
describe('maxMindDownloadRequest', () => {
|
|
57
|
+
it('uses the legacy query URL without an account id', () => {
|
|
58
|
+
const request = maxMindDownloadRequest({ licenseKey: 'secret-key' })
|
|
59
|
+
expect(request.url).toContain('edition_id=GeoLite2-Country')
|
|
60
|
+
expect(request.url).toContain('license_key=secret-key')
|
|
61
|
+
expect(request.headers.Authorization).toBeUndefined()
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
it('uses basic auth permalink when account id is set', () => {
|
|
65
|
+
const request = maxMindDownloadRequest({ licenseKey: 'secret-key', accountId: '123456' })
|
|
66
|
+
expect(request.url).toBe('https://download.maxmind.com/geoip/databases/GeoLite2-Country/download?suffix=tar.gz')
|
|
67
|
+
expect(request.headers.Authorization).toBe(`Basic ${Buffer.from('123456:secret-key').toString('base64')}`)
|
|
68
|
+
})
|
|
69
|
+
})
|
|
70
|
+
|
|
71
|
+
describe('ensureGeoLite2Country', () => {
|
|
72
|
+
afterEach(restoreEnv)
|
|
73
|
+
|
|
74
|
+
it('skips download when the license key is missing', async () => {
|
|
75
|
+
const env = {}
|
|
76
|
+
let called = false
|
|
77
|
+
const dest = await ensureGeoLite2Country({
|
|
78
|
+
env,
|
|
79
|
+
fetchImpl: async () => {
|
|
80
|
+
called = true
|
|
81
|
+
return { ok: false, status: 500, arrayBuffer: async () => new ArrayBuffer(0) }
|
|
82
|
+
},
|
|
83
|
+
})
|
|
84
|
+
expect(dest).toBeNull()
|
|
85
|
+
expect(called).toBe(false)
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
it('returns the existing file without fetching', async () => {
|
|
89
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'geolite2-'))
|
|
90
|
+
const dest = path.join(dir, 'GeoLite2-Country.mmdb')
|
|
91
|
+
fs.writeFileSync(dest, 'already-there')
|
|
92
|
+
const env = { GEOLITE2_COUNTRY_MMDB: dest, MAXMIND_LICENSE_KEY: 'secret-key' }
|
|
93
|
+
let called = false
|
|
94
|
+
const result = await ensureGeoLite2Country({
|
|
95
|
+
env,
|
|
96
|
+
fetchImpl: async () => {
|
|
97
|
+
called = true
|
|
98
|
+
return { ok: true, status: 200, arrayBuffer: async () => new ArrayBuffer(0) }
|
|
99
|
+
},
|
|
100
|
+
})
|
|
101
|
+
expect(result).toBe(dest)
|
|
102
|
+
expect(called).toBe(false)
|
|
103
|
+
fs.rmSync(dir, { recursive: true, force: true })
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
it('writes the extracted mmdb and sets GEOLITE2_COUNTRY_MMDB', async () => {
|
|
107
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'geolite2-'))
|
|
108
|
+
const dest = path.join(dir, 'GeoLite2-Country.mmdb')
|
|
109
|
+
const payload = Buffer.from('country-mmdb')
|
|
110
|
+
const archive = makeUstarTarGz('GeoLite2-Country_20240101/GeoLite2-Country.mmdb', payload)
|
|
111
|
+
const env = { GEOLITE2_COUNTRY_MMDB: dest, MAXMIND_LICENSE_KEY: 'secret-key' }
|
|
112
|
+
|
|
113
|
+
const result = await ensureGeoLite2Country({
|
|
114
|
+
env,
|
|
115
|
+
fetchImpl: async () => ({
|
|
116
|
+
ok: true,
|
|
117
|
+
status: 200,
|
|
118
|
+
arrayBuffer: async () => archive,
|
|
119
|
+
}),
|
|
120
|
+
})
|
|
121
|
+
|
|
122
|
+
expect(result).toBe(dest)
|
|
123
|
+
expect(env.GEOLITE2_COUNTRY_MMDB).toBe(dest)
|
|
124
|
+
expect(fs.readFileSync(dest).equals(payload)).toBe(true)
|
|
125
|
+
fs.rmSync(dir, { recursive: true, force: true })
|
|
126
|
+
})
|
|
127
|
+
|
|
128
|
+
it('does not throw when MaxMind returns an error', async () => {
|
|
129
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'geolite2-'))
|
|
130
|
+
const dest = path.join(dir, 'GeoLite2-Country.mmdb')
|
|
131
|
+
const env = { GEOLITE2_COUNTRY_MMDB: dest, MAXMIND_LICENSE_KEY: 'secret-key' }
|
|
132
|
+
const result = await ensureGeoLite2Country({
|
|
133
|
+
env,
|
|
134
|
+
fetchImpl: async () => ({ ok: false, status: 401, arrayBuffer: async () => new ArrayBuffer(0) }),
|
|
135
|
+
})
|
|
136
|
+
expect(result).toBeNull()
|
|
137
|
+
expect(fs.existsSync(dest)).toBe(false)
|
|
138
|
+
fs.rmSync(dir, { recursive: true, force: true })
|
|
139
|
+
})
|
|
140
|
+
})
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { gunzipSync } from 'node:zlib'
|
|
2
|
+
|
|
3
|
+
const BLOCK = 512
|
|
4
|
+
const MAX_ARCHIVE_BYTES = 50 * 1024 * 1024
|
|
5
|
+
const MAX_MMDB_BYTES = 20 * 1024 * 1024
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* @param {Buffer} header
|
|
9
|
+
* @param {number} start
|
|
10
|
+
* @param {number} len
|
|
11
|
+
* @returns {number}
|
|
12
|
+
*/
|
|
13
|
+
function readOctal (header, start, len) {
|
|
14
|
+
const raw = header.subarray(start, start + len).toString('utf8').replace(/\0/g, '').trim()
|
|
15
|
+
if (!raw) return 0
|
|
16
|
+
const parsed = Number.parseInt(raw, 8)
|
|
17
|
+
return Number.isFinite(parsed) ? parsed : 0
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* @param {Buffer} header
|
|
22
|
+
* @returns {string}
|
|
23
|
+
*/
|
|
24
|
+
function headerName (header) {
|
|
25
|
+
const name = header.subarray(0, 100).toString('utf8').replace(/\0/g, '').trim()
|
|
26
|
+
const prefix = header.subarray(345, 345 + 155).toString('utf8').replace(/\0/g, '').trim()
|
|
27
|
+
return prefix ? `${prefix}/${name}` : name
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Extract the first `.mmdb` member from a MaxMind GeoLite2 `.tar.gz`.
|
|
32
|
+
* Writes none of the tar paths — callers choose the destination (#804).
|
|
33
|
+
*
|
|
34
|
+
* @param {Buffer} gzipped
|
|
35
|
+
* @returns {Buffer}
|
|
36
|
+
*/
|
|
37
|
+
export function extractMmdbFromTarGz (gzipped) {
|
|
38
|
+
if (!Buffer.isBuffer(gzipped) || gzipped.length < 20) {
|
|
39
|
+
throw new Error('GeoLite2 archive is empty or invalid')
|
|
40
|
+
}
|
|
41
|
+
if (gzipped.length > MAX_ARCHIVE_BYTES) {
|
|
42
|
+
throw new Error('GeoLite2 archive is too large')
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const tar = gunzipSync(gzipped)
|
|
46
|
+
let offset = 0
|
|
47
|
+
while (offset + BLOCK <= tar.length) {
|
|
48
|
+
const header = tar.subarray(offset, offset + BLOCK)
|
|
49
|
+
if (header.every((byte) => byte === 0)) break
|
|
50
|
+
|
|
51
|
+
const size = readOctal(header, 124, 12)
|
|
52
|
+
const typeflag = String.fromCharCode(header[156] || 0)
|
|
53
|
+
const name = headerName(header)
|
|
54
|
+
offset += BLOCK
|
|
55
|
+
const padded = Math.ceil(size / BLOCK) * BLOCK
|
|
56
|
+
const content = tar.subarray(offset, Math.min(offset + size, tar.length))
|
|
57
|
+
offset += padded
|
|
58
|
+
|
|
59
|
+
const isFile = typeflag === '0' || typeflag === '\0' || typeflag === ''
|
|
60
|
+
if (!isFile || !name.toLowerCase().endsWith('.mmdb')) continue
|
|
61
|
+
if (content.length > MAX_MMDB_BYTES) {
|
|
62
|
+
throw new Error('GeoLite2 mmdb member is too large')
|
|
63
|
+
}
|
|
64
|
+
return Buffer.from(content)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
throw new Error('No .mmdb file in GeoLite2 archive')
|
|
68
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { extractMmdbFromTarGz } from './extract-mmdb-from-tar-gz.js'
|
|
2
|
+
import { makeUstarTarGz } from './tar-fixture.js'
|
|
3
|
+
|
|
4
|
+
describe('extractMmdbFromTarGz', () => {
|
|
5
|
+
it('returns the first .mmdb member', () => {
|
|
6
|
+
const payload = Buffer.from('fake-mmdb-bytes')
|
|
7
|
+
const archive = makeUstarTarGz('GeoLite2-Country_20240101/GeoLite2-Country.mmdb', payload)
|
|
8
|
+
expect(extractMmdbFromTarGz(archive).equals(payload)).toBe(true)
|
|
9
|
+
})
|
|
10
|
+
|
|
11
|
+
it('skips non-mmdb members', () => {
|
|
12
|
+
const readme = Buffer.from('not the database')
|
|
13
|
+
const archive = makeUstarTarGz('GeoLite2-Country_20240101/COPYRIGHT.txt', readme)
|
|
14
|
+
expect(() => extractMmdbFromTarGz(archive)).toThrow(/No \.mmdb file/)
|
|
15
|
+
})
|
|
16
|
+
|
|
17
|
+
it('rejects empty input', () => {
|
|
18
|
+
expect(() => extractMmdbFromTarGz(Buffer.alloc(0))).toThrow(/empty or invalid/)
|
|
19
|
+
})
|
|
20
|
+
})
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Well-known GeoLite2-Country path on platform containers (#804).
|
|
3
|
+
*
|
|
4
|
+
* Keep in sync with `@ossy/deployment-tools` `GEOLITE2_COUNTRY_MMDB_PATH`
|
|
5
|
+
* (ECS task-definition env). Writable on Fargate ephemeral disk.
|
|
6
|
+
*/
|
|
7
|
+
export const GEOLITE2_COUNTRY_MMDB_PATH = '/tmp/GeoLite2-Country.mmdb'
|
|
8
|
+
|
|
9
|
+
/** Abort hung MaxMind downloads before ECS health startPeriod (60s). */
|
|
10
|
+
export const GEOLITE2_COUNTRY_DOWNLOAD_TIMEOUT_MS = 30_000
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { gzipSync } from 'node:zlib'
|
|
2
|
+
|
|
3
|
+
const BLOCK = 512
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Build a gzipped ustar archive with a single file (test fixture).
|
|
7
|
+
*
|
|
8
|
+
* @param {string} name
|
|
9
|
+
* @param {Buffer | string} content
|
|
10
|
+
* @returns {Buffer}
|
|
11
|
+
*/
|
|
12
|
+
export function makeUstarTarGz (name, content) {
|
|
13
|
+
const data = Buffer.isBuffer(content) ? content : Buffer.from(content)
|
|
14
|
+
const header = Buffer.alloc(BLOCK)
|
|
15
|
+
header.write(name, 0, 100, 'utf8')
|
|
16
|
+
header.write('0000644\0', 100, 8, 'utf8')
|
|
17
|
+
header.write('0000000\0', 108, 8, 'utf8')
|
|
18
|
+
header.write('0000000\0', 116, 8, 'utf8')
|
|
19
|
+
header.write(`${data.length.toString(8).padStart(11, '0')}\0`, 124, 12, 'utf8')
|
|
20
|
+
header.write('00000000000\0', 136, 12, 'utf8')
|
|
21
|
+
header.write(' ', 148, 8, 'utf8')
|
|
22
|
+
header[156] = 0x30
|
|
23
|
+
header.write('ustar\0', 257, 6, 'utf8')
|
|
24
|
+
header.write('00', 263, 2, 'utf8')
|
|
25
|
+
let sum = 0
|
|
26
|
+
for (const byte of header) sum += byte
|
|
27
|
+
header.write(`${sum.toString(8).padStart(6, '0')}\0 `, 148, 8, 'utf8')
|
|
28
|
+
const padded = Buffer.alloc(Math.ceil(data.length / BLOCK) * BLOCK)
|
|
29
|
+
data.copy(padded)
|
|
30
|
+
return gzipSync(Buffer.concat([header, padded, Buffer.alloc(1024)]))
|
|
31
|
+
}
|
package/src/proxy-internal.js
CHANGED
|
@@ -14,6 +14,25 @@ function normalizeWorkspaceIdHeader (value) {
|
|
|
14
14
|
return first || undefined
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
+
/**
|
|
18
|
+
* Map `/@ossy…` to the same-process path (`/actions` or `/api/v0…`).
|
|
19
|
+
* @param {string} originalUrl
|
|
20
|
+
* @returns {string}
|
|
21
|
+
*/
|
|
22
|
+
export function ossyProxyLocalPath (originalUrl) {
|
|
23
|
+
const pathAfterOssy = String(originalUrl || '').replace(/^\/@ossy/, '') || '/'
|
|
24
|
+
const pathOnly = pathAfterOssy.split('?')[0]
|
|
25
|
+
if (pathOnly === '/actions' || pathOnly === '/events') {
|
|
26
|
+
return pathAfterOssy.startsWith('/') ? pathAfterOssy : `/${pathAfterOssy}`
|
|
27
|
+
}
|
|
28
|
+
return `/api/v0${pathAfterOssy.startsWith('/') ? pathAfterOssy : `/${pathAfterOssy}`}`
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Absolute OSSY_API_URL → HTTP hop. Unset / relative → same Express app. */
|
|
32
|
+
export function isRemoteOssyApiUrl (envUrl = process.env.OSSY_API_URL) {
|
|
33
|
+
return /^https?:\/\//i.test(String(envUrl || '').trim())
|
|
34
|
+
}
|
|
35
|
+
|
|
17
36
|
export function ProxyInternal () {
|
|
18
37
|
return (req, res, next) => {
|
|
19
38
|
if (!req.originalUrl.startsWith('/@ossy')) {
|
|
@@ -47,15 +66,20 @@ export function ProxyInternal () {
|
|
|
47
66
|
return
|
|
48
67
|
}
|
|
49
68
|
|
|
69
|
+
if (!isRemoteOssyApiUrl()) {
|
|
70
|
+
const dest = ossyProxyLocalPath(req.originalUrl)
|
|
71
|
+
log.info(`[@ossy/platform][proxy] ${req.method} ${req.originalUrl} → ${dest} (same origin)`)
|
|
72
|
+
req.url = dest
|
|
73
|
+
req.originalUrl = dest
|
|
74
|
+
return next()
|
|
75
|
+
}
|
|
76
|
+
|
|
50
77
|
log.info(`[@ossy/platform][proxy] ${req.method} ${req.originalUrl}`)
|
|
51
78
|
|
|
52
|
-
const domain = (process.env.OSSY_API_URL
|
|
79
|
+
const domain = String(process.env.OSSY_API_URL).trim()
|
|
53
80
|
.replace(/\/api\/v0\/?$/, '')
|
|
54
81
|
.replace(/\/$/, '')
|
|
55
|
-
const
|
|
56
|
-
// POST /actions lives on the app server root — not under /api/v0.
|
|
57
|
-
const upstreamPath = pathAfterOssy === '/actions' ? '/actions' : `/api/v0${pathAfterOssy}`
|
|
58
|
-
const url = `${domain}${upstreamPath}`
|
|
82
|
+
const url = `${domain}${ossyProxyLocalPath(req.originalUrl)}`
|
|
59
83
|
const forwardedHeaders = JSON.parse(JSON.stringify(req.headers))
|
|
60
84
|
const workspaceId = normalizeWorkspaceIdHeader(req.get('workspaceId'))
|
|
61
85
|
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it, jest } from '@jest/globals'
|
|
2
|
+
import {
|
|
3
|
+
isRemoteOssyApiUrl,
|
|
4
|
+
ossyProxyLocalPath,
|
|
5
|
+
ProxyInternal,
|
|
6
|
+
} from './proxy-internal.js'
|
|
7
|
+
|
|
8
|
+
describe('ossyProxyLocalPath', () => {
|
|
9
|
+
it('maps REST under /@ossy onto /api/v0', () => {
|
|
10
|
+
expect(ossyProxyLocalPath('/@ossy/users/me')).toBe('/api/v0/users/me')
|
|
11
|
+
expect(ossyProxyLocalPath('/@ossy/apps/ask?domain=ossy.se'))
|
|
12
|
+
.toBe('/api/v0/apps/ask?domain=ossy.se')
|
|
13
|
+
})
|
|
14
|
+
|
|
15
|
+
it('keeps POST /actions and GET /events at the server root', () => {
|
|
16
|
+
expect(ossyProxyLocalPath('/@ossy/actions')).toBe('/actions')
|
|
17
|
+
expect(ossyProxyLocalPath('/@ossy/actions?x=1')).toBe('/actions?x=1')
|
|
18
|
+
expect(ossyProxyLocalPath('/@ossy/events')).toBe('/events')
|
|
19
|
+
})
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
describe('isRemoteOssyApiUrl', () => {
|
|
23
|
+
it('treats unset, empty, and relative values as same-origin', () => {
|
|
24
|
+
expect(isRemoteOssyApiUrl(undefined)).toBe(false)
|
|
25
|
+
expect(isRemoteOssyApiUrl('')).toBe(false)
|
|
26
|
+
expect(isRemoteOssyApiUrl('/api/v0')).toBe(false)
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
it('treats absolute http(s) as a remote hop', () => {
|
|
30
|
+
expect(isRemoteOssyApiUrl('https://ossy.se/api/v0')).toBe(true)
|
|
31
|
+
expect(isRemoteOssyApiUrl('http://localhost:3006')).toBe(true)
|
|
32
|
+
})
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
describe('ProxyInternal same-origin rewrite', () => {
|
|
36
|
+
const original = process.env.OSSY_API_URL
|
|
37
|
+
|
|
38
|
+
afterEach(() => {
|
|
39
|
+
if (original === undefined) delete process.env.OSSY_API_URL
|
|
40
|
+
else process.env.OSSY_API_URL = original
|
|
41
|
+
})
|
|
42
|
+
|
|
43
|
+
it('rewrites /@ossy onto this process when OSSY_API_URL is unset', () => {
|
|
44
|
+
delete process.env.OSSY_API_URL
|
|
45
|
+
const req = { originalUrl: '/@ossy/users/me', url: '/@ossy/users/me', method: 'GET' }
|
|
46
|
+
const next = jest.fn()
|
|
47
|
+
ProxyInternal()(req, {}, next)
|
|
48
|
+
expect(req.url).toBe('/api/v0/users/me')
|
|
49
|
+
expect(req.originalUrl).toBe('/api/v0/users/me')
|
|
50
|
+
expect(next).toHaveBeenCalledTimes(1)
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
it('rewrites /@ossy/actions onto POST /actions', () => {
|
|
54
|
+
delete process.env.OSSY_API_URL
|
|
55
|
+
const req = { originalUrl: '/@ossy/actions', url: '/@ossy/actions', method: 'POST' }
|
|
56
|
+
const next = jest.fn()
|
|
57
|
+
ProxyInternal()(req, {}, next)
|
|
58
|
+
expect(req.url).toBe('/actions')
|
|
59
|
+
expect(req.originalUrl).toBe('/actions')
|
|
60
|
+
expect(next).toHaveBeenCalledTimes(1)
|
|
61
|
+
})
|
|
62
|
+
})
|
package/src/runtime.js
CHANGED
|
@@ -71,13 +71,24 @@ async function getSiteContext (domain) {
|
|
|
71
71
|
*
|
|
72
72
|
* Environment variables:
|
|
73
73
|
* OSSY_API_KEY — API JWT for CMS reads (required)
|
|
74
|
-
* OSSY_API_URL — Override API base URL (default: https://
|
|
74
|
+
* OSSY_API_URL — Override API base URL (default: https://ossy.se/api/v0)
|
|
75
75
|
* PORT — Override listen port (default: 3000)
|
|
76
76
|
*/
|
|
77
77
|
export async function startRuntime ({ port } = {}) {
|
|
78
78
|
const resolvedPort = port ?? resolvePort()
|
|
79
79
|
|
|
80
80
|
const app = express()
|
|
81
|
+
// CloudFront → ALB → Node: hop count so `req.ip` is the viewer (#769).
|
|
82
|
+
// Default 2. Avoid `true` (leftmost XFF — spoofable). Override via OSSY_TRUST_PROXY_HOPS.
|
|
83
|
+
{
|
|
84
|
+
const raw = process.env.OSSY_TRUST_PROXY_HOPS
|
|
85
|
+
if (raw != null && String(raw).trim() !== '') {
|
|
86
|
+
const hops = Number(raw)
|
|
87
|
+
app.set('trust proxy', Number.isFinite(hops) && hops >= 0 ? hops : 2)
|
|
88
|
+
} else {
|
|
89
|
+
app.set('trust proxy', 2)
|
|
90
|
+
}
|
|
91
|
+
}
|
|
81
92
|
// Liveness for ALB/ECS — before site loading so probes never depend on CMS/domain.
|
|
82
93
|
// Prefer OSSY_SERVICE_NAME (ECS task env) when set.
|
|
83
94
|
mountHealthEndpoint(app)
|
|
@@ -97,7 +108,7 @@ export async function startRuntime ({ port } = {}) {
|
|
|
97
108
|
next()
|
|
98
109
|
})
|
|
99
110
|
|
|
100
|
-
// Proxy /@ossy/*
|
|
111
|
+
// Proxy leftover /@ossy/* onto this process (/api/v0, /actions) unless OSSY_API_URL is absolute.
|
|
101
112
|
app.use(ProxyInternal())
|
|
102
113
|
|
|
103
114
|
// Force a fresh site download without waiting for the TTL to expire
|
package/src/server.js
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
} from './entitlements/action-entitlement.js'
|
|
29
29
|
import { closePushSseConnections, mountPushSse } from './push/mount-push-sse.js'
|
|
30
30
|
import { mountHealthEndpoint } from './health.js'
|
|
31
|
+
import { ensureGeoLite2Country } from './geoip/ensure-geolite2-country.js'
|
|
31
32
|
import {
|
|
32
33
|
createSlowRequestLogger,
|
|
33
34
|
createTimedOperation,
|
|
@@ -289,6 +290,10 @@ export async function startServer (options = {}) {
|
|
|
289
290
|
|
|
290
291
|
TaskService.startScheduler()
|
|
291
292
|
|
|
293
|
+
// GeoLite2-Country for Host location projections (#804). Shared by every
|
|
294
|
+
// startServer image (ossy.se, plexus-sanitas, …). Missing key is non-fatal.
|
|
295
|
+
await ensureGeoLite2Country()
|
|
296
|
+
|
|
292
297
|
if (process.env.DB_URL) {
|
|
293
298
|
await ensureEventStoreIndexes().catch((err) => {
|
|
294
299
|
log.error('Failed to ensure MongoDB indexes — aggregate reads may be very slow', undefined, err)
|
|
@@ -326,6 +331,17 @@ export async function startServer (options = {}) {
|
|
|
326
331
|
}
|
|
327
332
|
|
|
328
333
|
const app = express()
|
|
334
|
+
// CloudFront → ALB → Node: hop count so `req.ip` is the viewer (#769).
|
|
335
|
+
// Default 2. Avoid `true` (leftmost XFF — spoofable). Override via OSSY_TRUST_PROXY_HOPS.
|
|
336
|
+
{
|
|
337
|
+
const raw = process.env.OSSY_TRUST_PROXY_HOPS
|
|
338
|
+
if (raw != null && String(raw).trim() !== '') {
|
|
339
|
+
const hops = Number(raw)
|
|
340
|
+
app.set('trust proxy', Number.isFinite(hops) && hops >= 0 ? hops : 2)
|
|
341
|
+
} else {
|
|
342
|
+
app.set('trust proxy', 2)
|
|
343
|
+
}
|
|
344
|
+
}
|
|
329
345
|
// Liveness for ALB/ECS — before auth so probes never depend on cookies/Mongo.
|
|
330
346
|
// Prefer OSSY_SERVICE_NAME (ECS task env) when set.
|
|
331
347
|
mountHealthEndpoint(app)
|
package/src/site-loader.js
CHANGED
|
@@ -10,7 +10,7 @@ const CACHE_TTL_MS = 5 * 60 * 1000 // 5 minutes
|
|
|
10
10
|
const DOWNLOAD_CONCURRENCY = 10
|
|
11
11
|
const BASE_DIR = path.join(os.tmpdir(), 'ossy-rt')
|
|
12
12
|
|
|
13
|
-
const API_URL = process.env.OSSY_API_URL || 'https://
|
|
13
|
+
const API_URL = process.env.OSSY_API_URL || 'https://ossy.se/api/v0'
|
|
14
14
|
const API_KEY = process.env.OSSY_API_KEY
|
|
15
15
|
|
|
16
16
|
/** @type {Map<string, { buildDir: string, workspaceId: string, loadedAt: number }>} */
|
|
@@ -4,6 +4,7 @@ import { IntegrationService } from '../integration.service.js'
|
|
|
4
4
|
import { createLogger } from '@ossy/observability'
|
|
5
5
|
import { isPrimaryTaskForAction, taskIdFromActionId } from '@ossy/schema'
|
|
6
6
|
import { TaskRunService } from '../audit/task-run-service.js'
|
|
7
|
+
import { resolveClientIp, runWithEventContext } from '@ossy/event-store'
|
|
7
8
|
|
|
8
9
|
const SCHEDULER_INTERVAL_MS = 60_000
|
|
9
10
|
const _serviceLog = createLogger('TaskService')
|
|
@@ -108,18 +109,20 @@ export class TaskService {
|
|
|
108
109
|
log: context.log ?? createLogger(id),
|
|
109
110
|
}
|
|
110
111
|
|
|
111
|
-
return
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
112
|
+
return runWithEventContext({ ip: resolveClientIp(context.req) }, () =>
|
|
113
|
+
TaskRunService.execute({
|
|
114
|
+
taskId: id,
|
|
115
|
+
handler: task.handler,
|
|
116
|
+
context: invokeContext,
|
|
117
|
+
trigger: 'invoke',
|
|
118
|
+
triggeredBy: {
|
|
119
|
+
channel: context.req ? undefined : 'api',
|
|
120
|
+
actionInvocationId: context.actionInvocationId ?? null,
|
|
121
|
+
},
|
|
122
|
+
executionEnv: 'ossy_server',
|
|
123
|
+
audit,
|
|
124
|
+
}),
|
|
125
|
+
)
|
|
123
126
|
}
|
|
124
127
|
|
|
125
128
|
/** @param {string} id */
|
package/src/test/flow-runner.js
CHANGED
|
@@ -26,6 +26,7 @@ import { assertDownloadFilenames, resolveDownloadTarget } from './flow-download.
|
|
|
26
26
|
import { resolveFilesTarget } from './flow-files.js'
|
|
27
27
|
import { resolvePressKey } from './flow-press.js'
|
|
28
28
|
import { resolveViewportSize } from './flow-viewport.js'
|
|
29
|
+
import { flowBelongsToShard, parseFlowShard, sortFlowsForShard } from './flow-shard.js'
|
|
29
30
|
|
|
30
31
|
export {
|
|
31
32
|
applyFormFieldOverrides,
|
|
@@ -39,6 +40,12 @@ export {
|
|
|
39
40
|
resolveActionMatchers,
|
|
40
41
|
resolveActionService,
|
|
41
42
|
} from './flow-action.js'
|
|
43
|
+
export {
|
|
44
|
+
flowBelongsToShard,
|
|
45
|
+
flowIdsForShard,
|
|
46
|
+
parseFlowShard,
|
|
47
|
+
sortFlowsForShard,
|
|
48
|
+
} from './flow-shard.js'
|
|
42
49
|
export { resolveClickTarget } from './flow-click.js'
|
|
43
50
|
export {
|
|
44
51
|
assertDownloadFilenames,
|
|
@@ -840,17 +847,32 @@ export function registerFlow (mod, options = {}) {
|
|
|
840
847
|
/**
|
|
841
848
|
* Register all flows listed in `manifest.flows[]`.
|
|
842
849
|
* @param {string} manifestPath Absolute path to build/manifest.json
|
|
850
|
+
* @param {{ shard?: string }} [options] Optional shard override (`N/M`); defaults to `E2E_FLOW_SHARD`
|
|
843
851
|
*/
|
|
844
|
-
export async function registerFlowsFromManifest (manifestPath) {
|
|
852
|
+
export async function registerFlowsFromManifest (manifestPath, options = {}) {
|
|
845
853
|
const manifest = loadManifest(manifestPath)
|
|
846
854
|
const buildDir = path.dirname(manifestPath)
|
|
847
855
|
const staticDir = path.join(buildDir, 'public', 'static')
|
|
848
|
-
|
|
849
|
-
|
|
856
|
+
const shard = parseFlowShard(options.shard ?? process.env.E2E_FLOW_SHARD)
|
|
857
|
+
// Sort by stable flow id so independently built matrix jobs partition identically
|
|
858
|
+
// even when package discovery/`readdir` order differs (#768 / #770).
|
|
859
|
+
const flows = sortFlowsForShard(manifest.flows ?? [])
|
|
860
|
+
let registered = 0
|
|
861
|
+
|
|
862
|
+
for (let i = 0; i < flows.length; i++) {
|
|
863
|
+
if (!flowBelongsToShard(i, shard)) continue
|
|
864
|
+
const entry = flows[i]
|
|
850
865
|
const chunkName = entry.entry.replace(/^\/static\//, '')
|
|
851
866
|
const chunkPath = path.join(staticDir, chunkName)
|
|
852
867
|
const mod = await import(pathToFileURL(chunkPath).href + `?ts=${Date.now()}`)
|
|
853
868
|
registerFlow(mod, { manifestPath, locale: manifest.config?.defaultLanguage })
|
|
869
|
+
registered += 1
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
if (shard) {
|
|
873
|
+
console.log(
|
|
874
|
+
`[e2e] flow shard ${shard.index}/${shard.total}: registered ${registered}/${flows.length} flows`,
|
|
875
|
+
)
|
|
854
876
|
}
|
|
855
877
|
}
|
|
856
878
|
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CI flow partitioning helpers.
|
|
3
|
+
*
|
|
4
|
+
* All product flows register from one Playwright `runner.spec.js`, so Playwright's
|
|
5
|
+
* file-level `--shard` leaves shards 2..N empty. Matrix jobs set `E2E_FLOW_SHARD=N/M`
|
|
6
|
+
* and the flow runner registers only that slice.
|
|
7
|
+
*
|
|
8
|
+
* Membership is by **stable flow id order**, not discovery/`readdir` order — each
|
|
9
|
+
* matrix job builds independently (#768 / #770).
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Parse `E2E_FLOW_SHARD` / explicit shard string (`1/4`) into 1-based index + total.
|
|
14
|
+
*
|
|
15
|
+
* @param {string | undefined | null} value
|
|
16
|
+
* @returns {{ index: number, total: number } | null}
|
|
17
|
+
*/
|
|
18
|
+
export function parseFlowShard (value = process.env.E2E_FLOW_SHARD) {
|
|
19
|
+
if (value == null || value === '') return null
|
|
20
|
+
const match = String(value).trim().match(/^(\d+)\s*\/\s*(\d+)$/)
|
|
21
|
+
if (!match) {
|
|
22
|
+
throw new Error(`Invalid E2E_FLOW_SHARD "${value}" (expected N/M, e.g. 1/4)`)
|
|
23
|
+
}
|
|
24
|
+
const index = Number(match[1])
|
|
25
|
+
const total = Number(match[2])
|
|
26
|
+
if (!Number.isInteger(index) || !Number.isInteger(total) || total < 1 || index < 1 || index > total) {
|
|
27
|
+
throw new Error(`Invalid E2E_FLOW_SHARD "${value}" (index must be 1..total)`)
|
|
28
|
+
}
|
|
29
|
+
return { index, total }
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Deterministic order for shard partitioning (localeCompare on flow id).
|
|
34
|
+
*
|
|
35
|
+
* @param {Array<{ id?: string }>} flows
|
|
36
|
+
* @returns {Array<{ id?: string }>}
|
|
37
|
+
*/
|
|
38
|
+
export function sortFlowsForShard (flows) {
|
|
39
|
+
return [...(flows ?? [])].sort((a, b) =>
|
|
40
|
+
String(a?.id ?? '').localeCompare(String(b?.id ?? '')),
|
|
41
|
+
)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Stable round-robin membership for CI flow matrix jobs.
|
|
46
|
+
* @param {number} flowIndex 0-based position in the **id-sorted** flow list
|
|
47
|
+
* @param {{ index: number, total: number } | null} shard
|
|
48
|
+
*/
|
|
49
|
+
export function flowBelongsToShard (flowIndex, shard) {
|
|
50
|
+
if (!shard) return true
|
|
51
|
+
return flowIndex % shard.total === shard.index - 1
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Flow ids owned by a shard for a given (unordered) manifest list.
|
|
56
|
+
*
|
|
57
|
+
* @param {Array<{ id?: string }>} flows
|
|
58
|
+
* @param {{ index: number, total: number } | null} shard
|
|
59
|
+
* @returns {string[]}
|
|
60
|
+
*/
|
|
61
|
+
export function flowIdsForShard (flows, shard) {
|
|
62
|
+
const ordered = sortFlowsForShard(flows)
|
|
63
|
+
return ordered
|
|
64
|
+
.filter((_, i) => flowBelongsToShard(i, shard))
|
|
65
|
+
.map((flow) => String(flow.id ?? ''))
|
|
66
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { describe, expect, it } from '@jest/globals'
|
|
2
|
+
import {
|
|
3
|
+
flowBelongsToShard,
|
|
4
|
+
flowIdsForShard,
|
|
5
|
+
parseFlowShard,
|
|
6
|
+
sortFlowsForShard,
|
|
7
|
+
} from './flow-shard.js'
|
|
8
|
+
|
|
9
|
+
describe('parseFlowShard', () => {
|
|
10
|
+
it('returns null for empty values', () => {
|
|
11
|
+
expect(parseFlowShard(undefined)).toBeNull()
|
|
12
|
+
expect(parseFlowShard(null)).toBeNull()
|
|
13
|
+
expect(parseFlowShard('')).toBeNull()
|
|
14
|
+
})
|
|
15
|
+
|
|
16
|
+
it('parses N/M', () => {
|
|
17
|
+
expect(parseFlowShard('1/4')).toEqual({ index: 1, total: 4 })
|
|
18
|
+
expect(parseFlowShard(' 3 / 4 ')).toEqual({ index: 3, total: 4 })
|
|
19
|
+
})
|
|
20
|
+
|
|
21
|
+
it('rejects invalid shapes', () => {
|
|
22
|
+
expect(() => parseFlowShard('4')).toThrow(/Invalid E2E_FLOW_SHARD/)
|
|
23
|
+
expect(() => parseFlowShard('0/4')).toThrow(/Invalid E2E_FLOW_SHARD/)
|
|
24
|
+
expect(() => parseFlowShard('5/4')).toThrow(/Invalid E2E_FLOW_SHARD/)
|
|
25
|
+
})
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
describe('flowBelongsToShard', () => {
|
|
29
|
+
it('keeps every flow when shard is null', () => {
|
|
30
|
+
expect(flowBelongsToShard(0, null)).toBe(true)
|
|
31
|
+
expect(flowBelongsToShard(7, null)).toBe(true)
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
it('round-robins indices across shards', () => {
|
|
35
|
+
const shard1 = { index: 1, total: 4 }
|
|
36
|
+
const shard2 = { index: 2, total: 4 }
|
|
37
|
+
const owned = [...Array(8).keys()].filter(i => flowBelongsToShard(i, shard1))
|
|
38
|
+
const owned2 = [...Array(8).keys()].filter(i => flowBelongsToShard(i, shard2))
|
|
39
|
+
expect(owned).toEqual([0, 4])
|
|
40
|
+
expect(owned2).toEqual([1, 5])
|
|
41
|
+
})
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
describe('sortFlowsForShard / flowIdsForShard', () => {
|
|
45
|
+
it('sorts by stable id regardless of input order', () => {
|
|
46
|
+
const a = [{ id: 'zeta' }, { id: 'alpha' }, { id: 'mu' }]
|
|
47
|
+
const b = [{ id: 'mu' }, { id: 'zeta' }, { id: 'alpha' }]
|
|
48
|
+
expect(sortFlowsForShard(a).map(f => f.id)).toEqual(['alpha', 'mu', 'zeta'])
|
|
49
|
+
expect(sortFlowsForShard(b).map(f => f.id)).toEqual(['alpha', 'mu', 'zeta'])
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
it('assigns the same ids to shard 1/4 for two different input orders', () => {
|
|
53
|
+
const shard = { index: 1, total: 4 }
|
|
54
|
+
const orderA = [
|
|
55
|
+
{ id: '@ossy/z/flows/c' },
|
|
56
|
+
{ id: '@ossy/a/flows/x' },
|
|
57
|
+
{ id: '@ossy/m/flows/y' },
|
|
58
|
+
{ id: '@ossy/b/flows/w' },
|
|
59
|
+
{ id: '@ossy/n/flows/v' },
|
|
60
|
+
{ id: '@ossy/c/flows/u' },
|
|
61
|
+
{ id: '@ossy/d/flows/t' },
|
|
62
|
+
{ id: '@ossy/e/flows/s' },
|
|
63
|
+
]
|
|
64
|
+
const orderB = [...orderA].reverse()
|
|
65
|
+
|
|
66
|
+
expect(flowIdsForShard(orderA, shard)).toEqual(flowIdsForShard(orderB, shard))
|
|
67
|
+
// id-sorted: a/x, b/w, c/u, d/t, e/s, m/y, n/v, z/c → shard 1 owns indices 0,4
|
|
68
|
+
expect(flowIdsForShard(orderA, shard)).toEqual([
|
|
69
|
+
'@ossy/a/flows/x',
|
|
70
|
+
'@ossy/e/flows/s',
|
|
71
|
+
])
|
|
72
|
+
})
|
|
73
|
+
})
|