@netgreener/runtime 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +28 -0
- package/TUTORIAL.md +495 -0
- package/dist/aggregator.d.ts +43 -0
- package/dist/aggregator.js +270 -0
- package/dist/analyze/analyzeGate.d.ts +44 -0
- package/dist/analyze/analyzeGate.js +114 -0
- package/dist/analyze/apiOverconsumptionDetector.d.ts +13 -0
- package/dist/analyze/apiOverconsumptionDetector.js +75 -0
- package/dist/analyze/astDetectors.d.ts +38 -0
- package/dist/analyze/astDetectors.js +566 -0
- package/dist/analyze/buildResourceFindings.d.ts +34 -0
- package/dist/analyze/buildResourceFindings.js +86 -0
- package/dist/analyze/cli.d.ts +20 -0
- package/dist/analyze/cli.js +186 -0
- package/dist/analyze/cpuResourceDetectors.d.ts +35 -0
- package/dist/analyze/cpuResourceDetectors.js +163 -0
- package/dist/analyze/heuristicScan.d.ts +61 -0
- package/dist/analyze/heuristicScan.js +91 -0
- package/dist/analyze/index.d.ts +15 -0
- package/dist/analyze/index.js +15 -0
- package/dist/analyze/loadTypescript.d.ts +8 -0
- package/dist/analyze/loadTypescript.js +23 -0
- package/dist/analyze/memoryIoDetectors.d.ts +15 -0
- package/dist/analyze/memoryIoDetectors.js +68 -0
- package/dist/analyze/preferAstDetectors.d.ts +18 -0
- package/dist/analyze/preferAstDetectors.js +91 -0
- package/dist/analyze/projectScan.d.ts +19 -0
- package/dist/analyze/projectScan.js +115 -0
- package/dist/analyze/resourceAdmission.d.ts +33 -0
- package/dist/analyze/resourceAdmission.js +91 -0
- package/dist/analyze/resourceFindingCandidate.d.ts +61 -0
- package/dist/analyze/resourceFindingCandidate.js +75 -0
- package/dist/analyze/retryAmplificationDetector.d.ts +11 -0
- package/dist/analyze/retryAmplificationDetector.js +55 -0
- package/dist/analyze/schemas/resource-finding.schema.json +352 -0
- package/dist/analyze/serviceDiscovery.d.ts +66 -0
- package/dist/analyze/serviceDiscovery.js +210 -0
- package/dist/analyze/unboundedParallelismDetector.d.ts +11 -0
- package/dist/analyze/unboundedParallelismDetector.js +57 -0
- package/dist/analyze/validateResourceFinding.d.ts +19 -0
- package/dist/analyze/validateResourceFinding.js +46 -0
- package/dist/bullmq.d.ts +33 -0
- package/dist/bullmq.js +106 -0
- package/dist/cli/helpText.d.ts +8 -0
- package/dist/cli/helpText.js +99 -0
- package/dist/cli/netgreener.d.ts +12 -0
- package/dist/cli/netgreener.js +65 -0
- package/dist/collectorIpc.d.ts +28 -0
- package/dist/collectorIpc.js +189 -0
- package/dist/collectorMetadata.d.ts +15 -0
- package/dist/collectorMetadata.js +33 -0
- package/dist/config.d.ts +41 -0
- package/dist/config.js +97 -0
- package/dist/contract/index.d.ts +1 -0
- package/dist/contract/index.js +1 -0
- package/dist/contract/observation-envelope.schema.json +331 -0
- package/dist/contract/observation-protobuf-view.schema.json +773 -0
- package/dist/contract/observation-v1.d.mts +42 -0
- package/dist/contract/observation-v1.mjs +555 -0
- package/dist/contract/observationIdentity.d.ts +11 -0
- package/dist/contract/observationIdentity.js +48 -0
- package/dist/contract/observationProjection.d.ts +3 -0
- package/dist/contract/observationProjection.js +29 -0
- package/dist/contract/validate.d.ts +20 -0
- package/dist/contract/validate.js +227 -0
- package/dist/dogfood/mp3WorkerGates.d.ts +54 -0
- package/dist/dogfood/mp3WorkerGates.js +86 -0
- package/dist/dogfood/retainDryRunArtifact.d.ts +45 -0
- package/dist/dogfood/retainDryRunArtifact.js +103 -0
- package/dist/dogfood/tenantDogfoodGates.d.ts +76 -0
- package/dist/dogfood/tenantDogfoodGates.js +206 -0
- package/dist/exporter.d.ts +56 -0
- package/dist/exporter.js +304 -0
- package/dist/express.d.ts +44 -0
- package/dist/express.js +118 -0
- package/dist/externalApiMeter.d.ts +72 -0
- package/dist/externalApiMeter.js +1167 -0
- package/dist/fastify.d.ts +53 -0
- package/dist/fastify.js +148 -0
- package/dist/index.d.ts +32 -0
- package/dist/index.js +31 -0
- package/dist/n4/deploymentMatrix.d.ts +22 -0
- package/dist/n4/deploymentMatrix.js +57 -0
- package/dist/n4/missingnessInventory.d.ts +22 -0
- package/dist/n4/missingnessInventory.js +72 -0
- package/dist/nest.d.ts +42 -0
- package/dist/nest.js +87 -0
- package/dist/processResources.d.ts +28 -0
- package/dist/processResources.js +31 -0
- package/dist/processRuntime.d.ts +35 -0
- package/dist/processRuntime.js +96 -0
- package/dist/requestContext.d.ts +21 -0
- package/dist/requestContext.js +33 -0
- package/dist/runtime.d.ts +60 -0
- package/dist/runtime.js +289 -0
- package/dist/runtimeHealth.d.ts +35 -0
- package/dist/runtimeHealth.js +121 -0
- package/dist/serverlessHints.d.ts +26 -0
- package/dist/serverlessHints.js +52 -0
- package/dist/tenantContext.d.ts +78 -0
- package/dist/tenantContext.js +226 -0
- package/dist/tenantDefaults.d.ts +9 -0
- package/dist/tenantDefaults.js +9 -0
- package/dist/types.d.ts +131 -0
- package/dist/types.js +2 -0
- package/dist/uploader.d.ts +23 -0
- package/dist/uploader.js +52 -0
- package/dist/windowId.d.ts +2 -0
- package/dist/windowId.js +7 -0
- package/examples/analyze-manifest.mjs +30 -0
- package/examples/bullmq-live-smoke.mjs +178 -0
- package/examples/bullmq-mp3-worker-gate.mjs +237 -0
- package/examples/express-dry-run.mjs +72 -0
- package/examples/process-mp3-restart-gate.mjs +135 -0
- package/examples/retain-dry-run.mjs +110 -0
- package/examples/tenant-dogfood.mjs +269 -0
- package/fixtures/analyze-sample/architectureNoise.ts +13 -0
- package/fixtures/analyze-sample/cpuHotPaths.ts +27 -0
- package/fixtures/analyze-sample/fanOutClient.ts +15 -0
- package/fixtures/analyze-sample/inferenceHotPaths.ts +22 -0
- package/fixtures/analyze-sample/memoryIoHotPaths.ts +21 -0
- package/fixtures/analyze-sample/modelLoadHotPaths.ts +11 -0
- package/fixtures/analyze-sample/nestedLookup.ts +15 -0
- package/fixtures/analyze-sample/retryClient.ts +10 -0
- package/fixtures/analyze-sample/securitySmell.ts +8 -0
- package/fixtures/analyze-sample/server.ts +11 -0
- package/fixtures/analyze-sample/styleOnly.ts +7 -0
- package/fixtures/analyze-sample/worker.ts +4 -0
- package/fixtures/dual-view-projection.v1.json +53 -0
- package/fixtures/identity-preimages.v1.json +42 -0
- package/fixtures/mp2_shadow_digest_golden.v1.json +47 -0
- package/fixtures/projection-batch-document.v1.json +170 -0
- package/fixtures/projection-batch.v1.json +9 -0
- package/fixtures/python_reference_shapes.md +24 -0
- package/fixtures/session_metadata_runtime_v0.json +120 -0
- package/fixtures/session_metadata_with_tenant.json +85 -0
- package/package.json +137 -0
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MP3 process/cron restart gate (experimental).
|
|
3
|
+
*
|
|
4
|
+
* Proves ``startProcessRuntime`` + ``runWithProcessTenant`` survive a runtime
|
|
5
|
+
* reset (simulating process restart) and still emit ``node_process`` metadata.
|
|
6
|
+
* No Redis required. Default export mode remains direct.
|
|
7
|
+
*
|
|
8
|
+
* Dry-run (default):
|
|
9
|
+
* npm run example:process-mp3-gate
|
|
10
|
+
*
|
|
11
|
+
* Live (authorized only):
|
|
12
|
+
* $env:NETGREENER_RUNTIME_DRY_RUN="0"
|
|
13
|
+
* $env:NETGREENER_TOKEN="ngs_..."
|
|
14
|
+
* $env:NETGREENER_PROJECT_ID="97"
|
|
15
|
+
* $env:NETGREENER_API_URL="http://127.0.0.1:8000"
|
|
16
|
+
* npm run example:process-mp3-gate
|
|
17
|
+
*
|
|
18
|
+
* Does NOT claim MP3 production support or npm publish.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import {
|
|
22
|
+
_resetOutboundInstrumentationForTests,
|
|
23
|
+
_resetRuntimeForTests,
|
|
24
|
+
runWithProcessTenant,
|
|
25
|
+
startProcessRuntime,
|
|
26
|
+
flushProcessRuntime,
|
|
27
|
+
} from '../dist/index.js'
|
|
28
|
+
import {
|
|
29
|
+
extractPersistedRunId,
|
|
30
|
+
sessionMetadataForDogfood,
|
|
31
|
+
} from '../dist/dogfood/tenantDogfoodGates.js'
|
|
32
|
+
import {
|
|
33
|
+
assertMp3ProcessRestartGate,
|
|
34
|
+
formatMp3GateFailure,
|
|
35
|
+
} from '../dist/dogfood/mp3WorkerGates.js'
|
|
36
|
+
|
|
37
|
+
const dryRun =
|
|
38
|
+
(process.env.NETGREENER_RUNTIME_DRY_RUN || '1').trim() !== '0' &&
|
|
39
|
+
(process.env.NETGREENER_RUNTIME_DRY_RUN || '1').trim().toLowerCase() !== 'false'
|
|
40
|
+
const live = !dryRun
|
|
41
|
+
|
|
42
|
+
process.env.NETGREENER_SERVICE_RUNTIME = '1'
|
|
43
|
+
process.env.NETGREENER_TOKEN = process.env.NETGREENER_TOKEN || 'ngs_local_process_mp3'
|
|
44
|
+
process.env.NETGREENER_PROJECT_ID = process.env.NETGREENER_PROJECT_ID || '97'
|
|
45
|
+
process.env.NETGREENER_RUNTIME_DRY_RUN = dryRun ? '1' : '0'
|
|
46
|
+
process.env.NETGREENER_API_URL =
|
|
47
|
+
process.env.NETGREENER_API_URL || 'http://127.0.0.1:8000'
|
|
48
|
+
delete process.env.NETGREENER_EXPORT_MODE
|
|
49
|
+
|
|
50
|
+
const tenant = 'org_process_mp3'
|
|
51
|
+
const taskUnit = 'task:cron_job'
|
|
52
|
+
const generations = Math.max(2, Number(process.env.NETGREENER_PROCESS_GENERATIONS || '2'))
|
|
53
|
+
const jobsPerGen = Math.max(2, Number(process.env.NETGREENER_PROCESS_JOBS || '2'))
|
|
54
|
+
|
|
55
|
+
async function runGeneration(label) {
|
|
56
|
+
_resetRuntimeForTests()
|
|
57
|
+
_resetOutboundInstrumentationForTests()
|
|
58
|
+
const flushed = []
|
|
59
|
+
const runtime = startProcessRuntime({
|
|
60
|
+
onFlush: (result, payload) => {
|
|
61
|
+
flushed.push({ result, payload })
|
|
62
|
+
},
|
|
63
|
+
})
|
|
64
|
+
let completed = 0
|
|
65
|
+
for (let i = 0; i < jobsPerGen; i += 1) {
|
|
66
|
+
await runWithProcessTenant(
|
|
67
|
+
tenant,
|
|
68
|
+
async () => {
|
|
69
|
+
completed += 1
|
|
70
|
+
return { ok: true, i }
|
|
71
|
+
},
|
|
72
|
+
{ runtime, serviceUnit: taskUnit, source: 'manual' },
|
|
73
|
+
)
|
|
74
|
+
}
|
|
75
|
+
const flushResult = await flushProcessRuntime('manual')
|
|
76
|
+
if (!flushResult) {
|
|
77
|
+
throw new Error(`${label}: flush returned null`)
|
|
78
|
+
}
|
|
79
|
+
const flushPayload = flushed.at(-1)?.payload
|
|
80
|
+
const { sessionMetadata, uploadGate } = sessionMetadataForDogfood({
|
|
81
|
+
live,
|
|
82
|
+
flushResult,
|
|
83
|
+
flushPayload,
|
|
84
|
+
label,
|
|
85
|
+
})
|
|
86
|
+
if (!uploadGate.ok) {
|
|
87
|
+
throw new Error(`${label}: ${uploadGate.reason}`)
|
|
88
|
+
}
|
|
89
|
+
const runId = live ? extractPersistedRunId(flushResult.body) : null
|
|
90
|
+
return {
|
|
91
|
+
completed,
|
|
92
|
+
sessionMetadata,
|
|
93
|
+
runId,
|
|
94
|
+
windowId: flushPayload?.runtime_window_id,
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
let exitCode = 0
|
|
99
|
+
try {
|
|
100
|
+
console.log(
|
|
101
|
+
`MP3 process gate mode=${live ? 'LIVE' : 'DRY-RUN'} generations=${generations} jobsPerGen=${jobsPerGen}`,
|
|
102
|
+
)
|
|
103
|
+
for (let g = 0; g < generations; g += 1) {
|
|
104
|
+
const label = `mp3-process-gen-${g}`
|
|
105
|
+
const out = await runGeneration(label)
|
|
106
|
+
const gate = assertMp3ProcessRestartGate({
|
|
107
|
+
phase: 'process_restart',
|
|
108
|
+
completedJobs: out.completed,
|
|
109
|
+
minCompletedJobs: jobsPerGen,
|
|
110
|
+
sessionMetadata: out.sessionMetadata,
|
|
111
|
+
expectedTenant: tenant,
|
|
112
|
+
expectedTaskUnit: taskUnit,
|
|
113
|
+
})
|
|
114
|
+
console.log(`${formatMp3GateFailure(gate)} (generation=${g})`)
|
|
115
|
+
if (!gate.ok) exitCode = 1
|
|
116
|
+
if (live && out.runId != null) {
|
|
117
|
+
console.log(`[${label}] run_id=${out.runId} window=${out.windowId ?? 'n/a'}`)
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (exitCode === 0) {
|
|
121
|
+
console.log(
|
|
122
|
+
live
|
|
123
|
+
? 'MP3 PROCESS RESTART GATE PASS (generations + live upload)'
|
|
124
|
+
: 'MP3 PROCESS RESTART GATE PASS (generations; dry-run)',
|
|
125
|
+
)
|
|
126
|
+
console.log('Not claimed: MP3 production support label, managed sidecar, npm publish')
|
|
127
|
+
}
|
|
128
|
+
} catch (err) {
|
|
129
|
+
console.error('MP3 PROCESS RESTART GATE FAIL', err)
|
|
130
|
+
exitCode = 1
|
|
131
|
+
} finally {
|
|
132
|
+
_resetRuntimeForTests()
|
|
133
|
+
_resetOutboundInstrumentationForTests()
|
|
134
|
+
process.exitCode = exitCode
|
|
135
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run Express dry-run flush and retain a revision-bound redacted artifact.
|
|
3
|
+
* Forces DRY_RUN=1 — never uploads. Does not change default flush pipelines.
|
|
4
|
+
*
|
|
5
|
+
* npm run example:retain-dry-run
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { createServer } from 'node:http'
|
|
9
|
+
import { readFileSync } from 'node:fs'
|
|
10
|
+
import { dirname, join } from 'node:path'
|
|
11
|
+
import { fileURLToPath } from 'node:url'
|
|
12
|
+
import { execFileSync } from 'node:child_process'
|
|
13
|
+
|
|
14
|
+
import {
|
|
15
|
+
getExpressRuntime,
|
|
16
|
+
loadRuntimeConfig,
|
|
17
|
+
netgreenerExpressMiddleware,
|
|
18
|
+
_resetRuntimeForTests,
|
|
19
|
+
} from '../dist/index.js'
|
|
20
|
+
import {
|
|
21
|
+
assertArtifactRedacted,
|
|
22
|
+
buildDryRunArtifact,
|
|
23
|
+
writeDryRunArtifact,
|
|
24
|
+
} from '../dist/dogfood/retainDryRunArtifact.js'
|
|
25
|
+
|
|
26
|
+
const root = join(dirname(fileURLToPath(import.meta.url)), '..')
|
|
27
|
+
const artifactsDir = join(root, 'artifacts', 'dry-run')
|
|
28
|
+
|
|
29
|
+
function gitSha() {
|
|
30
|
+
try {
|
|
31
|
+
return execFileSync('git', ['rev-parse', 'HEAD'], {
|
|
32
|
+
cwd: root,
|
|
33
|
+
encoding: 'utf8',
|
|
34
|
+
}).trim()
|
|
35
|
+
} catch {
|
|
36
|
+
return 'unknown'
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const pkg = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8'))
|
|
41
|
+
|
|
42
|
+
_resetRuntimeForTests()
|
|
43
|
+
|
|
44
|
+
const config = loadRuntimeConfig({
|
|
45
|
+
...process.env,
|
|
46
|
+
NETGREENER_SERVICE_RUNTIME: '1',
|
|
47
|
+
NETGREENER_TOKEN: process.env.NETGREENER_TOKEN || 'ngs_local_dry_run_artifact',
|
|
48
|
+
NETGREENER_PROJECT_ID: process.env.NETGREENER_PROJECT_ID || '1',
|
|
49
|
+
NETGREENER_RUNTIME_DRY_RUN: '1',
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
const mw = netgreenerExpressMiddleware({
|
|
53
|
+
config,
|
|
54
|
+
fetchImpl: async () => {
|
|
55
|
+
throw new Error('retain-dry-run attempted an outbound API request')
|
|
56
|
+
},
|
|
57
|
+
})
|
|
58
|
+
|
|
59
|
+
const server = createServer((req, res) => {
|
|
60
|
+
const fakeReq = {
|
|
61
|
+
method: req.method,
|
|
62
|
+
path: (req.url || '/').split('?')[0],
|
|
63
|
+
url: req.url,
|
|
64
|
+
}
|
|
65
|
+
const fakeRes = {
|
|
66
|
+
statusCode: 200,
|
|
67
|
+
on(event, fn) {
|
|
68
|
+
if (event === 'finish' || event === 'close') {
|
|
69
|
+
res.on(event, fn)
|
|
70
|
+
}
|
|
71
|
+
return this
|
|
72
|
+
},
|
|
73
|
+
}
|
|
74
|
+
mw(fakeReq, fakeRes, () => {
|
|
75
|
+
res.statusCode = fakeReq.path === '/fail' ? 500 : 200
|
|
76
|
+
fakeRes.statusCode = res.statusCode
|
|
77
|
+
res.end(JSON.stringify({ path: fakeReq.path, status: res.statusCode }))
|
|
78
|
+
})
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
await new Promise((resolve) => server.listen(0, resolve))
|
|
82
|
+
const { port } = server.address()
|
|
83
|
+
await fetch(`http://127.0.0.1:${port}/health`)
|
|
84
|
+
await fetch(`http://127.0.0.1:${port}/v1/predict`, { method: 'POST' })
|
|
85
|
+
await fetch(`http://127.0.0.1:${port}/fail`)
|
|
86
|
+
const result = await getExpressRuntime().flush('manual')
|
|
87
|
+
server.close()
|
|
88
|
+
|
|
89
|
+
if (!result?.ok || !result.dryRun) {
|
|
90
|
+
throw new Error('retain-dry-run did not return a dry-run flush result')
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const artifact = buildDryRunArtifact({
|
|
94
|
+
payload: result.body,
|
|
95
|
+
gitSha: gitSha(),
|
|
96
|
+
packageName: pkg.name,
|
|
97
|
+
packageVersion: pkg.version,
|
|
98
|
+
sources: ['express-dry-run', 'example:retain-dry-run'],
|
|
99
|
+
})
|
|
100
|
+
assertArtifactRedacted(artifact)
|
|
101
|
+
if (!artifact.checks.ok) {
|
|
102
|
+
throw new Error('dry-run artifact checks failed (missing session_metadata / service_runtime_v0)')
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const { summaryPath, latestPath } = writeDryRunArtifact(artifactsDir, artifact)
|
|
106
|
+
console.log(
|
|
107
|
+
`DRY-RUN ARTIFACT RETAINED (sha=${artifact.git_sha_short}, collector=${artifact.checks.collector}, upload=never)`,
|
|
108
|
+
)
|
|
109
|
+
console.log(` summary: ${summaryPath}`)
|
|
110
|
+
console.log(` latest: ${latestPath}`)
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tenant + outbound E2E dogfood (experimental Node v0 / N2–N3).
|
|
3
|
+
*
|
|
4
|
+
* Dry-run (default — no network to api_server):
|
|
5
|
+
*
|
|
6
|
+
* cd netgreener_node
|
|
7
|
+
* npm run example:dogfood-tenant
|
|
8
|
+
*
|
|
9
|
+
* Live upload to your api_server (you supply secrets):
|
|
10
|
+
*
|
|
11
|
+
* $env:NETGREENER_RUNTIME_DRY_RUN="0"
|
|
12
|
+
* $env:NETGREENER_API_URL="http://127.0.0.1:8000" # or staging / any non-vendor host
|
|
13
|
+
* $env:NETGREENER_TOKEN="ngs_..."
|
|
14
|
+
* $env:NETGREENER_PROJECT_ID="123"
|
|
15
|
+
* npm run example:dogfood-tenant
|
|
16
|
+
*
|
|
17
|
+
* Live mode requires real HTTP uploads to succeed. Failed uploads fail the run;
|
|
18
|
+
* local payload capture is never a substitute for a successful live upload.
|
|
19
|
+
*
|
|
20
|
+
* Then check dashboard: External API Impact → Client company (org_acme / org_beta).
|
|
21
|
+
*
|
|
22
|
+
* This script does NOT require Redis/BullMQ. It uses the BullMQ processor wrap
|
|
23
|
+
* with a job-like object, and stubs only vendor fetch hosts (Azure DI / OpenAI).
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { createServer } from 'node:http'
|
|
27
|
+
|
|
28
|
+
import {
|
|
29
|
+
_resetOutboundInstrumentationForTests,
|
|
30
|
+
_resetRuntimeForTests,
|
|
31
|
+
getBullMqRuntime,
|
|
32
|
+
getExpressRuntime,
|
|
33
|
+
loadRuntimeConfig,
|
|
34
|
+
netgreenerBullMqProcessor,
|
|
35
|
+
netgreenerExpressMiddleware,
|
|
36
|
+
validateExternalApiV0,
|
|
37
|
+
validateRuntimeSessionMetadata,
|
|
38
|
+
} from '../dist/index.js'
|
|
39
|
+
import {
|
|
40
|
+
createVendorAwareFetch,
|
|
41
|
+
extractPersistedRunId,
|
|
42
|
+
sessionMetadataForDogfood,
|
|
43
|
+
} from '../dist/dogfood/tenantDogfoodGates.js'
|
|
44
|
+
|
|
45
|
+
_resetRuntimeForTests()
|
|
46
|
+
_resetOutboundInstrumentationForTests()
|
|
47
|
+
|
|
48
|
+
const dryRun =
|
|
49
|
+
(process.env.NETGREENER_RUNTIME_DRY_RUN || '1').trim() !== '0' &&
|
|
50
|
+
(process.env.NETGREENER_RUNTIME_DRY_RUN || '1').trim().toLowerCase() !== 'false'
|
|
51
|
+
const live = !dryRun
|
|
52
|
+
|
|
53
|
+
process.env.NETGREENER_SERVICE_RUNTIME = '1'
|
|
54
|
+
process.env.NETGREENER_TOKEN = process.env.NETGREENER_TOKEN || 'ngs_local_dogfood'
|
|
55
|
+
process.env.NETGREENER_PROJECT_ID = process.env.NETGREENER_PROJECT_ID || '1'
|
|
56
|
+
process.env.NETGREENER_RUNTIME_DRY_RUN = dryRun ? '1' : '0'
|
|
57
|
+
process.env.NETGREENER_TENANT_SOURCE = process.env.NETGREENER_TENANT_SOURCE || 'header'
|
|
58
|
+
process.env.NETGREENER_TENANT_HEADER =
|
|
59
|
+
process.env.NETGREENER_TENANT_HEADER || 'X-Organization-Id'
|
|
60
|
+
process.env.NETGREENER_API_URL =
|
|
61
|
+
process.env.NETGREENER_API_URL || 'http://127.0.0.1:8000'
|
|
62
|
+
|
|
63
|
+
const config = loadRuntimeConfig()
|
|
64
|
+
|
|
65
|
+
/** Stub vendor hostnames only; configured API destination always uses real HTTP. */
|
|
66
|
+
const realFetch = globalThis.fetch.bind(globalThis)
|
|
67
|
+
globalThis.fetch = createVendorAwareFetch(realFetch, { apiBaseUrl: config.apiUrl })
|
|
68
|
+
|
|
69
|
+
const flushed = []
|
|
70
|
+
|
|
71
|
+
const mw = netgreenerExpressMiddleware({
|
|
72
|
+
config,
|
|
73
|
+
onFlush: (result, payload) => {
|
|
74
|
+
flushed.push({ result, payload })
|
|
75
|
+
const mode = result.ok
|
|
76
|
+
? result.dryRun
|
|
77
|
+
? 'dry-run ok'
|
|
78
|
+
: `HTTP ${result.status}`
|
|
79
|
+
: `FAIL ${result.error}`
|
|
80
|
+
console.log(`[flush express] ${mode}`)
|
|
81
|
+
},
|
|
82
|
+
})
|
|
83
|
+
|
|
84
|
+
function assertDogfood(label, sessionMetadata) {
|
|
85
|
+
const contract = validateRuntimeSessionMetadata(sessionMetadata)
|
|
86
|
+
if (!contract.ok) {
|
|
87
|
+
console.error(`[${label}] contract failed`, contract.issues)
|
|
88
|
+
return false
|
|
89
|
+
}
|
|
90
|
+
const ext = sessionMetadata.external_api_v0
|
|
91
|
+
if (!ext) {
|
|
92
|
+
console.error(`[${label}] missing external_api_v0`)
|
|
93
|
+
return false
|
|
94
|
+
}
|
|
95
|
+
const extOk = validateExternalApiV0(ext)
|
|
96
|
+
if (!extOk.ok) {
|
|
97
|
+
console.error(`[${label}] external_api_v0 invalid`, extOk.issues)
|
|
98
|
+
return false
|
|
99
|
+
}
|
|
100
|
+
const byTenant = ext.by_tenant || {}
|
|
101
|
+
const keys = Object.keys(byTenant)
|
|
102
|
+
if (!keys.length) {
|
|
103
|
+
console.error(`[${label}] external_api_v0.by_tenant empty`)
|
|
104
|
+
return false
|
|
105
|
+
}
|
|
106
|
+
console.log(`[${label}] by_tenant keys:`, keys.join(', '))
|
|
107
|
+
for (const k of keys) {
|
|
108
|
+
const row = byTenant[k]
|
|
109
|
+
console.log(
|
|
110
|
+
` - ${k}: calls=${row.calls} errors=${row.errors} providers=${(row.providers || [])
|
|
111
|
+
.map((p) => p.provider_key)
|
|
112
|
+
.join('|')}`,
|
|
113
|
+
)
|
|
114
|
+
}
|
|
115
|
+
return true
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function resolveWindow(label, flushResult, flushPayload) {
|
|
119
|
+
const { sessionMetadata, uploadGate } = sessionMetadataForDogfood({
|
|
120
|
+
live,
|
|
121
|
+
flushResult,
|
|
122
|
+
flushPayload,
|
|
123
|
+
label,
|
|
124
|
+
})
|
|
125
|
+
if (!uploadGate.ok) {
|
|
126
|
+
console.error(`[${label}] ${uploadGate.reason}`)
|
|
127
|
+
return { ok: false, sessionMetadata: null }
|
|
128
|
+
}
|
|
129
|
+
if (live) {
|
|
130
|
+
const runId = extractPersistedRunId(flushResult?.body)
|
|
131
|
+
const windowId = flushPayload?.runtime_window_id
|
|
132
|
+
console.log(
|
|
133
|
+
`[${label}] live upload ok status=${flushResult.status} run_id=${runId ?? 'n/a'} runtime_window_id=${windowId ?? 'n/a'}`,
|
|
134
|
+
)
|
|
135
|
+
}
|
|
136
|
+
return { ok: true, sessionMetadata }
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const server = createServer((req, res) => {
|
|
140
|
+
const path = (req.url || '/').split('?')[0]
|
|
141
|
+
const fakeReq = {
|
|
142
|
+
method: req.method,
|
|
143
|
+
path,
|
|
144
|
+
url: req.url,
|
|
145
|
+
headers: req.headers,
|
|
146
|
+
route: path.startsWith('/v1/') ? { path: path.replace(/\/[^/]+$/, '/:id').includes(':') ? path : path } : undefined,
|
|
147
|
+
}
|
|
148
|
+
if (path === '/v1/documents/ocr') {
|
|
149
|
+
fakeReq.route = { path: '/v1/documents/ocr' }
|
|
150
|
+
}
|
|
151
|
+
const fakeRes = {
|
|
152
|
+
statusCode: 200,
|
|
153
|
+
on(event, fn) {
|
|
154
|
+
if (event === 'finish' || event === 'close') res.on(event, fn)
|
|
155
|
+
return this
|
|
156
|
+
},
|
|
157
|
+
}
|
|
158
|
+
mw(fakeReq, fakeRes, async () => {
|
|
159
|
+
try {
|
|
160
|
+
await fetch('https://eastus.api.cognitive.microsoft.com/formrecognizer/v2.1/prebuilt/receipt/analyze', {
|
|
161
|
+
method: 'POST',
|
|
162
|
+
})
|
|
163
|
+
await fetch('https://api.openai.com/v1/chat/completions', { method: 'POST' })
|
|
164
|
+
res.statusCode = 200
|
|
165
|
+
fakeRes.statusCode = 200
|
|
166
|
+
res.end(JSON.stringify({ ok: true, path }))
|
|
167
|
+
} catch (err) {
|
|
168
|
+
res.statusCode = 500
|
|
169
|
+
fakeRes.statusCode = 500
|
|
170
|
+
res.end(JSON.stringify({ error: String(err) }))
|
|
171
|
+
}
|
|
172
|
+
})
|
|
173
|
+
})
|
|
174
|
+
|
|
175
|
+
server.listen(0, async () => {
|
|
176
|
+
const { port } = server.address()
|
|
177
|
+
console.log(`dogfood server http://127.0.0.1:${port}`)
|
|
178
|
+
console.log(`mode: ${dryRun ? 'DRY-RUN (no api_server upload)' : 'LIVE upload (real HTTP; fail on upload error)'}`)
|
|
179
|
+
console.log(`api: ${config.apiUrl} project: ${config.projectId}`)
|
|
180
|
+
|
|
181
|
+
let ok = true
|
|
182
|
+
|
|
183
|
+
await fetch(`http://127.0.0.1:${port}/v1/documents/ocr`, {
|
|
184
|
+
method: 'POST',
|
|
185
|
+
headers: { 'X-Organization-Id': 'org_acme' },
|
|
186
|
+
})
|
|
187
|
+
|
|
188
|
+
const expressFlush = await getExpressRuntime().flush('manual')
|
|
189
|
+
const expressResolved = resolveWindow('express+fetch', expressFlush, flushed.at(-1)?.payload)
|
|
190
|
+
if (!expressResolved.ok || !expressResolved.sessionMetadata) {
|
|
191
|
+
ok = false
|
|
192
|
+
} else {
|
|
193
|
+
const expressMeta = expressResolved.sessionMetadata
|
|
194
|
+
ok = assertDogfood('express+fetch', expressMeta) && ok
|
|
195
|
+
if (!expressMeta.external_api_v0?.by_tenant?.org_acme) {
|
|
196
|
+
console.error('expected org_acme in external_api_v0.by_tenant')
|
|
197
|
+
ok = false
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
process.env.NETGREENER_TENANT_SOURCE = 'task_kwarg'
|
|
202
|
+
process.env.NETGREENER_TENANT_TASK_KWARG = 'organization_id'
|
|
203
|
+
_resetRuntimeForTests()
|
|
204
|
+
|
|
205
|
+
const workerFlushed = []
|
|
206
|
+
const processor = netgreenerBullMqProcessor(
|
|
207
|
+
async (job) => {
|
|
208
|
+
await fetch('https://eastus.api.cognitive.microsoft.com/formrecognizer/documentModels/prebuilt-read:analyze', {
|
|
209
|
+
method: 'POST',
|
|
210
|
+
})
|
|
211
|
+
return { document_id: job.data.document_id }
|
|
212
|
+
},
|
|
213
|
+
{
|
|
214
|
+
onFlush: (result, payload) => {
|
|
215
|
+
workerFlushed.push({ result, payload })
|
|
216
|
+
console.log(
|
|
217
|
+
`[flush bullmq] ${result.ok ? (result.dryRun ? 'dry-run ok' : `HTTP ${result.status}`) : result.error}`,
|
|
218
|
+
)
|
|
219
|
+
},
|
|
220
|
+
},
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
await processor({
|
|
224
|
+
id: 'job-1',
|
|
225
|
+
name: 'analyze_document',
|
|
226
|
+
data: { organization_id: 'org_beta', document_id: 'doc-99' },
|
|
227
|
+
})
|
|
228
|
+
|
|
229
|
+
const workerFlush = await getBullMqRuntime().flush('manual')
|
|
230
|
+
const workerResolved = resolveWindow('bullmq-processor', workerFlush, workerFlushed.at(-1)?.payload)
|
|
231
|
+
if (!workerResolved.ok || !workerResolved.sessionMetadata) {
|
|
232
|
+
ok = false
|
|
233
|
+
} else {
|
|
234
|
+
const workerMeta = workerResolved.sessionMetadata
|
|
235
|
+
ok = assertDogfood('bullmq-processor', workerMeta) && ok
|
|
236
|
+
if (!workerMeta.external_api_v0?.by_tenant?.org_beta) {
|
|
237
|
+
console.error('expected org_beta in external_api_v0.by_tenant')
|
|
238
|
+
ok = false
|
|
239
|
+
}
|
|
240
|
+
const taskUnit = workerMeta.service_runtime_v0?.units?.find(
|
|
241
|
+
(u) => u.service_unit === 'task:analyze_document',
|
|
242
|
+
)
|
|
243
|
+
if (!taskUnit) {
|
|
244
|
+
console.error('expected service_runtime unit task:analyze_document')
|
|
245
|
+
ok = false
|
|
246
|
+
} else {
|
|
247
|
+
console.log(`[bullmq-processor] task unit calls=${taskUnit.calls} type=${taskUnit.unit_type}`)
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
console.log('\n--- summary ---')
|
|
252
|
+
if (ok) {
|
|
253
|
+
console.log(live ? 'DOGFOOD PASS (live uploads succeeded)' : 'DOGFOOD PASS (dry-run payload smoke)')
|
|
254
|
+
if (dryRun) {
|
|
255
|
+
console.log(
|
|
256
|
+
'Next: set NETGREENER_RUNTIME_DRY_RUN=0, TOKEN/PROJECT_ID/API_URL, re-run; live mode fails if upload fails.',
|
|
257
|
+
)
|
|
258
|
+
} else {
|
|
259
|
+
console.log(
|
|
260
|
+
'Correlate run_id / runtime_window_id above with RunSessions + External APIs UI for org_acme / org_beta.',
|
|
261
|
+
)
|
|
262
|
+
}
|
|
263
|
+
} else {
|
|
264
|
+
console.log('DOGFOOD FAIL — see errors above')
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
server.close()
|
|
268
|
+
process.exit(ok ? 0 : 1)
|
|
269
|
+
})
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Non-resource architecture noise — must not emit resource_finding_v1.
|
|
3
|
+
* Prefer hexagonal modules and avoid god classes.
|
|
4
|
+
*/
|
|
5
|
+
export abstract class BaseService {
|
|
6
|
+
abstract run(): Promise<void>
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export class OrchestratorService extends BaseService {
|
|
10
|
+
async run(): Promise<void> {
|
|
11
|
+
return
|
|
12
|
+
}
|
|
13
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { execSync } from 'node:child_process'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Fixture: sleep polling + regex compile + subprocess in loops (CPU RD0 twins).
|
|
5
|
+
*/
|
|
6
|
+
export function pollUntilReady(check: () => boolean): void {
|
|
7
|
+
while (!check()) {
|
|
8
|
+
setTimeout(() => {}, 50)
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export function scanLines(lines: string[], needle: string): number {
|
|
13
|
+
let hits = 0
|
|
14
|
+
let blob = ''
|
|
15
|
+
for (const line of lines) {
|
|
16
|
+
const re = new RegExp(needle, 'i')
|
|
17
|
+
if (re.test(line)) hits += 1
|
|
18
|
+
blob += `${line}`
|
|
19
|
+
}
|
|
20
|
+
return hits
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function runTools(cmds: string[]): void {
|
|
24
|
+
for (const cmd of cmds) {
|
|
25
|
+
execSync(cmd)
|
|
26
|
+
}
|
|
27
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fixture: API inside .map / Promise.all — overconsumption + unbounded_parallelism.
|
|
3
|
+
*/
|
|
4
|
+
export async function fetchAll(urls: string[]): Promise<Response[]> {
|
|
5
|
+
return Promise.all(urls.map((url) => fetch(url)))
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Fixture: API inside plain for-loop — external_api_overconsumption (network_call_in_loop).
|
|
10
|
+
*/
|
|
11
|
+
export async function fetchSequential(urls: string[]): Promise<void> {
|
|
12
|
+
for (const url of urls) {
|
|
13
|
+
await fetch(url)
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/** Fixture: model inference inside loop → per_item_instead_of_batch / inference_inside_loop */
|
|
2
|
+
export async function scorePerItem(
|
|
3
|
+
model: { predict: (x: unknown) => Promise<unknown> },
|
|
4
|
+
batches: unknown[],
|
|
5
|
+
): Promise<unknown[]> {
|
|
6
|
+
const out: unknown[] = []
|
|
7
|
+
for (const batch of batches) {
|
|
8
|
+
out.push(await model.predict(batch))
|
|
9
|
+
}
|
|
10
|
+
return out
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export async function runOnnxPerItem(
|
|
14
|
+
session: { run: (feeds: Record<string, unknown>) => Promise<unknown> },
|
|
15
|
+
feedsList: Record<string, unknown>[],
|
|
16
|
+
): Promise<unknown[]> {
|
|
17
|
+
const out: unknown[] = []
|
|
18
|
+
for (const feeds of feedsList) {
|
|
19
|
+
out.push(await session.run(feeds))
|
|
20
|
+
}
|
|
21
|
+
return out
|
|
22
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Fixture: repeated file reads in a loop + full JSON dataset materialization.
|
|
5
|
+
*/
|
|
6
|
+
export function loadManyConfigs(paths: string[]): string[] {
|
|
7
|
+
const out: string[] = []
|
|
8
|
+
for (const p of paths) {
|
|
9
|
+
out.push(readFileSync(p, 'utf8'))
|
|
10
|
+
}
|
|
11
|
+
return out
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function loadDataset(path: string): unknown {
|
|
15
|
+
const raw = readFileSync(path, 'utf8')
|
|
16
|
+
return JSON.parse(raw)
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function loadDatasetOneLiner(path: string): unknown {
|
|
20
|
+
return JSON.parse(readFileSync(path, 'utf8'))
|
|
21
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Fixture: model load inside loop → process_or_runtime_churn / model_load_in_loop */
|
|
2
|
+
export async function loadPerItem(paths: string[]): Promise<unknown[]> {
|
|
3
|
+
const out: unknown[] = []
|
|
4
|
+
for (const p of paths) {
|
|
5
|
+
const model = await loadLayersModel(`file://${p}`)
|
|
6
|
+
out.push(model)
|
|
7
|
+
}
|
|
8
|
+
return out
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
declare function loadLayersModel(url: string): Promise<unknown>
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** Fixture: nested-loop array lookup → per_item_instead_of_batch */
|
|
2
|
+
export function matchMany(
|
|
3
|
+
left: Array<{ id: string }>,
|
|
4
|
+
right: Array<{ id: string }>,
|
|
5
|
+
): Array<{ id: string }> {
|
|
6
|
+
const out: Array<{ id: string }> = []
|
|
7
|
+
for (const a of left) {
|
|
8
|
+
for (const b of right) {
|
|
9
|
+
if (right.find((x) => x.id === a.id)) {
|
|
10
|
+
out.push(b)
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
return out
|
|
15
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fixture: retry loop with fetch — should yield retry_amplification candidate.
|
|
3
|
+
*/
|
|
4
|
+
export async function submitWithRetries(url: string): Promise<Response> {
|
|
5
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
6
|
+
const res = await fetch(url)
|
|
7
|
+
if (res.ok) return res
|
|
8
|
+
}
|
|
9
|
+
throw new Error('exhausted retries')
|
|
10
|
+
}
|