@capybara-hq/kernel 0.1.0-rc.0 → 0.1.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bootstrap.json +10 -6
- package/dist/channels/outbound.js +2 -1
- package/dist/config/schema.d.ts +15 -1
- package/dist/config/schema.js +44 -3
- package/dist/conversations/titles.js +2 -1
- package/dist/ctx/queue.js +3 -2
- package/dist/errors.d.ts +1 -0
- package/dist/errors.js +1 -0
- package/dist/evals/worker.js +2 -1
- package/dist/extensions/bootstrap.d.ts +39 -12
- package/dist/extensions/bootstrap.js +132 -38
- package/dist/http/app.js +51 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.js +4 -2
- package/dist/kernel.d.ts +12 -9
- package/dist/kernel.js +7 -7
- package/dist/log-config.d.ts +9 -0
- package/dist/log-config.js +27 -0
- package/dist/log-context.d.ts +32 -0
- package/dist/log-context.js +40 -0
- package/dist/log-loki.d.ts +34 -0
- package/dist/log-loki.js +145 -0
- package/dist/log.d.ts +41 -10
- package/dist/log.js +91 -17
- package/dist/net/public-address.js +3 -2
- package/dist/runtime/maintenance.js +2 -1
- package/dist/runtime/run-worker.js +2 -1
- package/dist/runtime/usage-report.js +2 -1
- package/dist/tools/builtins/sandbox/common.d.ts +5 -3
- package/dist/tools/builtins/sandbox/common.js +5 -3
- package/package.json +4 -2
package/dist/index.d.ts
CHANGED
|
@@ -28,7 +28,7 @@ export { ExtensionAdminOps, type ExtensionAdminOpsDeps } from "./extensions/admi
|
|
|
28
28
|
export { ExtensionAdmin, PLUGINS_CHANGED_TOPIC } from "./extensions/admin-view.js";
|
|
29
29
|
export { type ApprovalDecision, ExtensionApprovals, type PendingExtension, type PermissionSnapshot, planApprovals, snapshotCovers, withPreApproved, } from "./extensions/approvals.js";
|
|
30
30
|
export { blobKey, ExtensionBlobIntegrityError, type ExtensionBlobSource, ExtensionBlobs, } from "./extensions/blobs.js";
|
|
31
|
-
export { BOOTSTRAP_MANIFEST_FILE, type BootstrapEntryResult, BootstrapFailedError, type BootstrapFailure, type BootstrapResult, BootstrapService, type BootstrapServiceDeps, bootstrapManifestSha256, readBootstrapManifest, } from "./extensions/bootstrap.js";
|
|
31
|
+
export { BOOTSTRAP_MANIFEST_FILE, type BootstrapEntryResult, BootstrapFailedError, type BootstrapFailure, type BootstrapResolveOptions, type BootstrapResult, BootstrapService, type BootstrapServiceDeps, bootstrapManifestSha256, readBootstrapManifest, resolveBootstrapPackages, } from "./extensions/bootstrap.js";
|
|
32
32
|
export { type ExtensionCli, openExtensionCli } from "./extensions/cli.js";
|
|
33
33
|
export { DEPLOYMENT_SECRETS_ORG, type DeploymentConfigField, type DeploymentConfigView, ExtensionDeploymentConfigService, mergeDeploymentConfig, PLUGIN_DEPLOYMENT_CONFIG_TOPIC, readStoredDeploymentConfig, } from "./extensions/deployment-config.js";
|
|
34
34
|
export { assertSecretsReachable, ExtensionHost, type ExtensionLoadPlan, type InspectedExtension, type InstalledExtensionLoader, type InstallFailureReason, } from "./extensions/host.js";
|
|
@@ -53,7 +53,9 @@ export { DUMMY_PASSWORD_HASH, hashPassword, verifyPassword } from "./identity/pa
|
|
|
53
53
|
export { type IssuedSession, REFRESH_RACE_GRACE_MS, SessionService, } from "./identity/sessions.js";
|
|
54
54
|
export { DEPLOYMENT_SETTINGS, type DeploymentSettings, SettingsService, } from "./identity/settings.js";
|
|
55
55
|
export { createKernel, KERNEL_MIGRATIONS_DIR, KERNEL_SCHEMA, type Kernel, type KernelOptions, type KernelRuns, migrateAll, } from "./kernel.js";
|
|
56
|
-
export { createLogger, type KernelLogger, LOG_LEVELS, type LogLevel } from "./log.js";
|
|
56
|
+
export { createLogger, type KernelLogger, LOG_FORMATS, LOG_LEVELS, type LogFormat, type LoggerOptions, type LogLevel, type LogOutput, } from "./log.js";
|
|
57
|
+
export { createConfiguredLogger, loggerOptionsFromConfig } from "./log-config.js";
|
|
58
|
+
export { type ContextJob, type JobWrapper, type LogContext, logContext, setJobWrapper, setTraceContextSource, withJobContext, withLogContext, } from "./log-context.js";
|
|
57
59
|
export { type Org, OrgsService } from "./orgs/orgs.service.js";
|
|
58
60
|
export { fetchNpmPackage, IntegrityMismatchError, type NpmFetchConfig, NpmFetchError, PackageTooLargeError, } from "./packages/npm.js";
|
|
59
61
|
export { UNKNOWN_RESULT_TEXT } from "./runtime/context/pairing.js";
|
package/dist/index.js
CHANGED
|
@@ -27,7 +27,7 @@ export { ExtensionAdminOps } from "./extensions/admin-ops.js";
|
|
|
27
27
|
export { ExtensionAdmin, PLUGINS_CHANGED_TOPIC } from "./extensions/admin-view.js";
|
|
28
28
|
export { ExtensionApprovals, planApprovals, snapshotCovers, withPreApproved, } from "./extensions/approvals.js";
|
|
29
29
|
export { blobKey, ExtensionBlobIntegrityError, ExtensionBlobs, } from "./extensions/blobs.js";
|
|
30
|
-
export { BOOTSTRAP_MANIFEST_FILE, BootstrapFailedError, BootstrapService, bootstrapManifestSha256, readBootstrapManifest, } from "./extensions/bootstrap.js";
|
|
30
|
+
export { BOOTSTRAP_MANIFEST_FILE, BootstrapFailedError, BootstrapService, bootstrapManifestSha256, readBootstrapManifest, resolveBootstrapPackages, } from "./extensions/bootstrap.js";
|
|
31
31
|
export { openExtensionCli } from "./extensions/cli.js";
|
|
32
32
|
export { DEPLOYMENT_SECRETS_ORG, ExtensionDeploymentConfigService, mergeDeploymentConfig, PLUGIN_DEPLOYMENT_CONFIG_TOPIC, readStoredDeploymentConfig, } from "./extensions/deployment-config.js";
|
|
33
33
|
export { assertSecretsReachable, ExtensionHost, } from "./extensions/host.js";
|
|
@@ -52,7 +52,9 @@ export { DUMMY_PASSWORD_HASH, hashPassword, verifyPassword } from "./identity/pa
|
|
|
52
52
|
export { REFRESH_RACE_GRACE_MS, SessionService, } from "./identity/sessions.js";
|
|
53
53
|
export { DEPLOYMENT_SETTINGS, SettingsService, } from "./identity/settings.js";
|
|
54
54
|
export { createKernel, KERNEL_MIGRATIONS_DIR, KERNEL_SCHEMA, migrateAll, } from "./kernel.js";
|
|
55
|
-
export { createLogger, LOG_LEVELS } from "./log.js";
|
|
55
|
+
export { createLogger, LOG_FORMATS, LOG_LEVELS, } from "./log.js";
|
|
56
|
+
export { createConfiguredLogger, loggerOptionsFromConfig } from "./log-config.js";
|
|
57
|
+
export { logContext, setJobWrapper, setTraceContextSource, withJobContext, withLogContext, } from "./log-context.js";
|
|
56
58
|
export { OrgsService } from "./orgs/orgs.service.js";
|
|
57
59
|
export { fetchNpmPackage, IntegrityMismatchError, NpmFetchError, PackageTooLargeError, } from "./packages/npm.js";
|
|
58
60
|
export { UNKNOWN_RESULT_TEXT } from "./runtime/context/pairing.js";
|
package/dist/kernel.d.ts
CHANGED
|
@@ -25,7 +25,7 @@ import { type KernelHttp } from "./http/app.js";
|
|
|
25
25
|
import { IdentityBinding } from "./identity/binding.js";
|
|
26
26
|
import { ExternalIdentities } from "./identity/external.js";
|
|
27
27
|
import { SsoFlow } from "./identity/sso.js";
|
|
28
|
-
import {
|
|
28
|
+
import type { KernelLogger } from "./log.js";
|
|
29
29
|
import { PricesService } from "./models/prices.js";
|
|
30
30
|
import { ModelPurposesService } from "./models/purposes.js";
|
|
31
31
|
import { DirectorySync } from "./orgs/directory-sync.js";
|
|
@@ -153,7 +153,7 @@ export interface Kernel {
|
|
|
153
153
|
readonly catalog: CatalogService;
|
|
154
154
|
/** Enterprise import of plugins (plugin tgz files): prepare / confirm and staging cleanup. */
|
|
155
155
|
readonly pluginImport: PluginImport;
|
|
156
|
-
/** The bootstrap step (package ecosystem spec §7); `start()` runs it when given `
|
|
156
|
+
/** The bootstrap step (package ecosystem spec §7); `start()` runs it when given `bootstrap`. */
|
|
157
157
|
readonly bootstrap: BootstrapService;
|
|
158
158
|
/** Marketplaces: official / enterprise marketplaces and their refresh. */
|
|
159
159
|
readonly marketplaces: MarketplacesService;
|
|
@@ -209,7 +209,7 @@ export interface KernelOptions {
|
|
|
209
209
|
source?: ConfigSource;
|
|
210
210
|
/** 解析插件裸包名的目录,默认 `process.cwd()`;相对路径按当前 cwd 解析。 */
|
|
211
211
|
resolveFrom?: string;
|
|
212
|
-
/**
|
|
212
|
+
/** Defaults to a logger built from `config.log` and `config.roles` (createConfiguredLogger). */
|
|
213
213
|
logger?: KernelLogger;
|
|
214
214
|
/** 热更新时与来源原始值合并的环境变量,默认 `process.env`。 */
|
|
215
215
|
env?: NodeJS.ProcessEnv;
|
|
@@ -223,13 +223,16 @@ export interface KernelOptions {
|
|
|
223
223
|
/** 评测整轮回放的测试钩子。 */
|
|
224
224
|
evalHooks?: EvalReplayHooks;
|
|
225
225
|
/**
|
|
226
|
-
* The
|
|
227
|
-
*
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
*
|
|
226
|
+
* The bootstrap step (package ecosystem spec §7, startup bootstrap spec). When given, `start()`
|
|
227
|
+
* runs it (see `extensions/bootstrap.ts`) before loading extensions unless it is already done for
|
|
228
|
+
* the manifest, and refuses to start when it cannot complete a required entry. `dir`: the
|
|
229
|
+
* optional offline directory whose `*.tgz` files are matched to the entries by sha512 (a missing
|
|
230
|
+
* directory is an empty one); entries without a matching file are downloaded from
|
|
231
|
+
* `packages.npmRegistry`. Omitted: no bootstrap step (tests, unless they ask for it).
|
|
231
232
|
*/
|
|
232
|
-
|
|
233
|
+
bootstrap?: {
|
|
234
|
+
dir?: string;
|
|
235
|
+
};
|
|
233
236
|
/** The bootstrap manifest; default the `bootstrap.json` shipped with the kernel package. */
|
|
234
237
|
bootstrapManifest?: BootstrapManifest;
|
|
235
238
|
/**
|
package/dist/kernel.js
CHANGED
|
@@ -47,7 +47,7 @@ import { ExternalIdentities } from "./identity/external.js";
|
|
|
47
47
|
import { SessionService } from "./identity/sessions.js";
|
|
48
48
|
import { SettingsService } from "./identity/settings.js";
|
|
49
49
|
import { SsoFlow } from "./identity/sso.js";
|
|
50
|
-
import {
|
|
50
|
+
import { createConfiguredLogger } from "./log-config.js";
|
|
51
51
|
import { PricesService } from "./models/prices.js";
|
|
52
52
|
import { ModelPurposesService } from "./models/purposes.js";
|
|
53
53
|
import { DirectorySync } from "./orgs/directory-sync.js";
|
|
@@ -168,7 +168,7 @@ export async function createKernel(config, opts = {}) {
|
|
|
168
168
|
throw new Error(`config field ${p} is marked hot but has no applier`);
|
|
169
169
|
}
|
|
170
170
|
const env = opts.env ?? process.env;
|
|
171
|
-
const log = opts.logger ??
|
|
171
|
+
const log = opts.logger ?? createConfiguredLogger(config);
|
|
172
172
|
log.setLevel(config.log.level);
|
|
173
173
|
log.addSecrets(secretValues(KernelConfigSchema, config));
|
|
174
174
|
let current = config;
|
|
@@ -375,7 +375,7 @@ export async function createKernel(config, opts = {}) {
|
|
|
375
375
|
// The second pass records what the first one did (not `kept` for what it installed)
|
|
376
376
|
await bootstrap.run({
|
|
377
377
|
manifest,
|
|
378
|
-
dir,
|
|
378
|
+
...(dir !== undefined && { dir }),
|
|
379
379
|
signal: startAbort.signal,
|
|
380
380
|
...(bootstrapCarry !== undefined && { carry: bootstrapCarry }),
|
|
381
381
|
});
|
|
@@ -1133,12 +1133,12 @@ export async function createKernel(config, opts = {}) {
|
|
|
1133
1133
|
// official 清单来源按配置同步(幂等;每个副本启动时都做)
|
|
1134
1134
|
await marketplaces.syncOfficialFromConfig();
|
|
1135
1135
|
// The bootstrap plugins (spec §7), once per manifest across replicas (advisory lock)
|
|
1136
|
-
if (opts.
|
|
1136
|
+
if (opts.bootstrap !== undefined) {
|
|
1137
1137
|
configDeclared = await ExtensionHost.declaredNames(config.extensions, {
|
|
1138
1138
|
...(opts.resolveFrom !== undefined && { resolveFrom: opts.resolveFrom }),
|
|
1139
1139
|
});
|
|
1140
1140
|
}
|
|
1141
|
-
const bootstrapAgain = opts.
|
|
1141
|
+
const bootstrapAgain = opts.bootstrap !== undefined && (await runBootstrap(opts.bootstrap.dir));
|
|
1142
1142
|
checkpoint();
|
|
1143
1143
|
// 配置声明 → 安装记录(C15 / C13)。安装记录的包在本副本同步到 extensionHost.dir 后导入,
|
|
1144
1144
|
// 与运行中的 reconcile 走同一条路径;迁移由一个副本持锁执行(ensureMigrated,C16)
|
|
@@ -1291,8 +1291,8 @@ export async function createKernel(config, opts = {}) {
|
|
|
1291
1291
|
checkpoint();
|
|
1292
1292
|
await extensionLifecycle.reconcile("startup");
|
|
1293
1293
|
checkpoint();
|
|
1294
|
-
if (bootstrapAgain && opts.
|
|
1295
|
-
await runBootstrap(opts.
|
|
1294
|
+
if (bootstrapAgain && opts.bootstrap !== undefined) {
|
|
1295
|
+
await runBootstrap(opts.bootstrap.dir, true);
|
|
1296
1296
|
checkpoint();
|
|
1297
1297
|
}
|
|
1298
1298
|
if (opts.source?.watch) {
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { KernelConfig } from "./config/schema.js";
|
|
2
|
+
import { type KernelLogger, type LoggerOptions } from "./log.js";
|
|
3
|
+
/**
|
|
4
|
+
* Logger options from the kernel config: the console format, `role` (the process's roles joined
|
|
5
|
+
* by `,`) on every record, and the Loki output when `log.loki.url` is set. `extra` overrides (tests).
|
|
6
|
+
*/
|
|
7
|
+
export declare function loggerOptionsFromConfig(config: KernelConfig, extra?: Partial<LoggerOptions>): LoggerOptions;
|
|
8
|
+
/** The process logger as configured by `log.*` and `roles`. */
|
|
9
|
+
export declare function createConfiguredLogger(config: KernelConfig): KernelLogger;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { createLogger } from "./log.js";
|
|
2
|
+
/**
|
|
3
|
+
* Logger options from the kernel config: the console format, `role` (the process's roles joined
|
|
4
|
+
* by `,`) on every record, and the Loki output when `log.loki.url` is set. `extra` overrides (tests).
|
|
5
|
+
*/
|
|
6
|
+
export function loggerOptionsFromConfig(config, extra = {}) {
|
|
7
|
+
const role = config.roles.join(",");
|
|
8
|
+
const { url, username, password, labels } = config.log.loki;
|
|
9
|
+
return {
|
|
10
|
+
format: config.log.format,
|
|
11
|
+
base: { role },
|
|
12
|
+
...(url !== undefined && {
|
|
13
|
+
loki: {
|
|
14
|
+
url,
|
|
15
|
+
labels,
|
|
16
|
+
role,
|
|
17
|
+
...(username !== undefined && { username }),
|
|
18
|
+
...(password !== undefined && { password }),
|
|
19
|
+
},
|
|
20
|
+
}),
|
|
21
|
+
...extra,
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
/** The process logger as configured by `log.*` and `roles`. */
|
|
25
|
+
export function createConfiguredLogger(config) {
|
|
26
|
+
return createLogger(config.log.level, loggerOptionsFromConfig(config));
|
|
27
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fields every log record written inside the current async context carries (`reqId` for an HTTP
|
|
3
|
+
* request, `queue` / `jobId` / `runId` for a queue job). Read by the logger's pino mixin; the
|
|
4
|
+
* context is per async call chain, never shared between requests.
|
|
5
|
+
*/
|
|
6
|
+
export type LogContext = Readonly<Record<string, string>>;
|
|
7
|
+
/** A source of extra per-record fields, e.g. the active trace / span ids when tracing is on. */
|
|
8
|
+
type ContextSource = () => Record<string, string> | undefined;
|
|
9
|
+
/** The fields of the current context (plus the trace source's), or undefined outside any. */
|
|
10
|
+
export declare function logContext(): LogContext | undefined;
|
|
11
|
+
/**
|
|
12
|
+
* Registers the source of trace fields (`traceId` / `spanId`); set by the tracing bootstrap only
|
|
13
|
+
* when tracing is enabled, so the kernel never imports OpenTelemetry itself.
|
|
14
|
+
*/
|
|
15
|
+
export declare function setTraceContextSource(source: ContextSource | undefined): void;
|
|
16
|
+
/** Runs `fn` with `fields` added to the current context (undefined values are skipped). */
|
|
17
|
+
export declare function withLogContext<T>(fields: Record<string, string | undefined>, fn: () => T): T;
|
|
18
|
+
/** The part of a BullMQ job the context reads. */
|
|
19
|
+
export interface ContextJob {
|
|
20
|
+
id?: string | undefined;
|
|
21
|
+
data?: unknown;
|
|
22
|
+
}
|
|
23
|
+
/** Hooks around every queue job (the tracing bootstrap wraps jobs in a span through this). */
|
|
24
|
+
export type JobWrapper = <T>(queue: string, job: ContextJob, fn: () => Promise<T>) => Promise<T>;
|
|
25
|
+
/** Registers a wrapper run around every job processor (tracing); undefined removes it. */
|
|
26
|
+
export declare function setJobWrapper(wrapper: JobWrapper | undefined): void;
|
|
27
|
+
/**
|
|
28
|
+
* Runs a queue job's processor with `queue`, `jobId` and, when the job data has one, `runId` in the
|
|
29
|
+
* log context (and inside the registered job wrapper, if any).
|
|
30
|
+
*/
|
|
31
|
+
export declare function withJobContext<T>(queue: string, job: ContextJob, fn: () => Promise<T>): Promise<T>;
|
|
32
|
+
export {};
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
const storage = new AsyncLocalStorage();
|
|
3
|
+
let traceSource;
|
|
4
|
+
/** The fields of the current context (plus the trace source's), or undefined outside any. */
|
|
5
|
+
export function logContext() {
|
|
6
|
+
const own = storage.getStore();
|
|
7
|
+
const trace = traceSource?.();
|
|
8
|
+
if (!trace)
|
|
9
|
+
return own;
|
|
10
|
+
return own ? { ...own, ...trace } : trace;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Registers the source of trace fields (`traceId` / `spanId`); set by the tracing bootstrap only
|
|
14
|
+
* when tracing is enabled, so the kernel never imports OpenTelemetry itself.
|
|
15
|
+
*/
|
|
16
|
+
export function setTraceContextSource(source) {
|
|
17
|
+
traceSource = source;
|
|
18
|
+
}
|
|
19
|
+
/** Runs `fn` with `fields` added to the current context (undefined values are skipped). */
|
|
20
|
+
export function withLogContext(fields, fn) {
|
|
21
|
+
const next = { ...storage.getStore() };
|
|
22
|
+
for (const [k, v] of Object.entries(fields))
|
|
23
|
+
if (v !== undefined)
|
|
24
|
+
next[k] = v;
|
|
25
|
+
return storage.run(next, fn);
|
|
26
|
+
}
|
|
27
|
+
let jobWrapper;
|
|
28
|
+
/** Registers a wrapper run around every job processor (tracing); undefined removes it. */
|
|
29
|
+
export function setJobWrapper(wrapper) {
|
|
30
|
+
jobWrapper = wrapper;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Runs a queue job's processor with `queue`, `jobId` and, when the job data has one, `runId` in the
|
|
34
|
+
* log context (and inside the registered job wrapper, if any).
|
|
35
|
+
*/
|
|
36
|
+
export function withJobContext(queue, job, fn) {
|
|
37
|
+
const data = job.data;
|
|
38
|
+
const runId = typeof data?.runId === "string" ? data.runId : undefined;
|
|
39
|
+
return withLogContext({ queue, jobId: job.id, runId }, () => jobWrapper ? jobWrapper(queue, job, fn) : fn());
|
|
40
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/** Loki output of the kernel logger (`log.loki.*`). */
|
|
2
|
+
export interface LokiOptions {
|
|
3
|
+
/** Loki base URL (`http://loki:3100`) or the full push URL (`…/loki/api/v1/push`). */
|
|
4
|
+
url: string;
|
|
5
|
+
username?: string;
|
|
6
|
+
password?: string;
|
|
7
|
+
/** Extra stream labels; `role` and `level` are always the record's own. */
|
|
8
|
+
labels: Record<string, string>;
|
|
9
|
+
/** The `role` label: the process's roles joined by `,`. */
|
|
10
|
+
role: string;
|
|
11
|
+
/** Push interval in ms; default 2000. */
|
|
12
|
+
intervalMs?: number;
|
|
13
|
+
/** Records kept while Loki is slow or down; the oldest are dropped beyond it. Default 10000. */
|
|
14
|
+
maxBuffer?: number;
|
|
15
|
+
/** Where push failures are reported (at most once per minute); default stderr. */
|
|
16
|
+
onError?: (message: string) => void;
|
|
17
|
+
/** Clock for the error rate limit (tests). */
|
|
18
|
+
now?: () => number;
|
|
19
|
+
}
|
|
20
|
+
/** A pino destination that batches records and pushes them to Loki. */
|
|
21
|
+
export interface LokiStream {
|
|
22
|
+
write(line: string): void;
|
|
23
|
+
/** Pushes everything buffered (waits for a push in flight first). Never rejects. */
|
|
24
|
+
flush(): Promise<void>;
|
|
25
|
+
/** Flushes and stops the interval. */
|
|
26
|
+
close(): Promise<void>;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Batches JSON log lines and POSTs them to Loki's push API (JSON body), one stream per level with
|
|
30
|
+
* the labels `{ job: "capybara", ...labels, role, level }`. Pushing never blocks or throws into the
|
|
31
|
+
* caller: a failed push drops its batch and is reported to `onError` at most once per minute; while
|
|
32
|
+
* Loki is down at most `maxBuffer` records are kept. The interval timer is unref'd.
|
|
33
|
+
*/
|
|
34
|
+
export declare function createLokiStream(opts: LokiOptions): LokiStream;
|
package/dist/log-loki.js
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
const PUSH_PATH = "/loki/api/v1/push";
|
|
2
|
+
const PUSH_TIMEOUT_MS = 10_000;
|
|
3
|
+
/** A batch is pushed right away once it reaches this many records. */
|
|
4
|
+
const BATCH_SIZE = 1_000;
|
|
5
|
+
const ERROR_NOTE_INTERVAL_MS = 60_000;
|
|
6
|
+
function pushUrl(url) {
|
|
7
|
+
const u = new URL(url);
|
|
8
|
+
if (!u.pathname.endsWith(PUSH_PATH))
|
|
9
|
+
u.pathname = `${u.pathname.replace(/\/+$/, "")}${PUSH_PATH}`;
|
|
10
|
+
return u;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Batches JSON log lines and POSTs them to Loki's push API (JSON body), one stream per level with
|
|
14
|
+
* the labels `{ job: "capybara", ...labels, role, level }`. Pushing never blocks or throws into the
|
|
15
|
+
* caller: a failed push drops its batch and is reported to `onError` at most once per minute; while
|
|
16
|
+
* Loki is down at most `maxBuffer` records are kept. The interval timer is unref'd.
|
|
17
|
+
*/
|
|
18
|
+
export function createLokiStream(opts) {
|
|
19
|
+
const url = pushUrl(opts.url);
|
|
20
|
+
const intervalMs = opts.intervalMs ?? 2_000;
|
|
21
|
+
const maxBuffer = opts.maxBuffer ?? 10_000;
|
|
22
|
+
const now = opts.now ?? Date.now;
|
|
23
|
+
const onError = opts.onError ?? ((m) => void process.stderr.write(`capybara: ${m}\n`));
|
|
24
|
+
// the marker keeps these pushes out of HTTP client traces (apps/capybara tracing.ts)
|
|
25
|
+
const headers = {
|
|
26
|
+
"content-type": "application/json",
|
|
27
|
+
"x-capybara-log-push": "1",
|
|
28
|
+
};
|
|
29
|
+
if (opts.username !== undefined || opts.password !== undefined) {
|
|
30
|
+
const token = Buffer.from(`${opts.username ?? ""}:${opts.password ?? ""}`).toString("base64");
|
|
31
|
+
headers.authorization = `Basic ${token}`;
|
|
32
|
+
}
|
|
33
|
+
const baseLabels = { job: "capybara", ...opts.labels, role: opts.role };
|
|
34
|
+
let buffer = [];
|
|
35
|
+
let dropped = 0;
|
|
36
|
+
let inFlight;
|
|
37
|
+
let timer;
|
|
38
|
+
let lastNote = Number.NEGATIVE_INFINITY;
|
|
39
|
+
let closed = false;
|
|
40
|
+
/** Reports at most once a minute; `message` names the Loki origin only (a URL may carry credentials). */
|
|
41
|
+
const report = (message) => {
|
|
42
|
+
const t = now();
|
|
43
|
+
if (t - lastNote < ERROR_NOTE_INTERVAL_MS)
|
|
44
|
+
return;
|
|
45
|
+
lastNote = t;
|
|
46
|
+
onError(`${message}; further Loki errors are not reported for a minute`);
|
|
47
|
+
};
|
|
48
|
+
const note = (reason) => report(`cannot push logs to Loki at ${url.origin} (${reason}); records are dropped until it recovers`);
|
|
49
|
+
const pushBatch = async (batch, lost) => {
|
|
50
|
+
const streams = new Map();
|
|
51
|
+
for (const e of batch) {
|
|
52
|
+
const values = streams.get(e.level) ?? [];
|
|
53
|
+
values.push([e.ns, e.line]);
|
|
54
|
+
streams.set(e.level, values);
|
|
55
|
+
}
|
|
56
|
+
const body = JSON.stringify({
|
|
57
|
+
streams: [...streams].map(([level, values]) => ({
|
|
58
|
+
stream: { ...baseLabels, level },
|
|
59
|
+
values,
|
|
60
|
+
})),
|
|
61
|
+
});
|
|
62
|
+
try {
|
|
63
|
+
const res = await fetch(url, {
|
|
64
|
+
method: "POST",
|
|
65
|
+
headers,
|
|
66
|
+
body,
|
|
67
|
+
signal: AbortSignal.timeout(PUSH_TIMEOUT_MS),
|
|
68
|
+
});
|
|
69
|
+
await res.body?.cancel();
|
|
70
|
+
if (!res.ok)
|
|
71
|
+
note(`HTTP ${res.status}`);
|
|
72
|
+
else if (lost > 0) {
|
|
73
|
+
report(`${lost} log records for Loki at ${url.origin} were dropped: the buffer was full`);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
catch (e) {
|
|
77
|
+
note(e instanceof Error ? (e.cause instanceof Error ? e.cause.message : e.message) : "error");
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
/** Starts a push of the current buffer unless one is in flight. */
|
|
81
|
+
const kick = () => {
|
|
82
|
+
if (inFlight || buffer.length === 0)
|
|
83
|
+
return inFlight ?? Promise.resolve();
|
|
84
|
+
const batch = buffer;
|
|
85
|
+
const lost = dropped;
|
|
86
|
+
buffer = [];
|
|
87
|
+
dropped = 0;
|
|
88
|
+
inFlight = pushBatch(batch, lost).finally(() => {
|
|
89
|
+
inFlight = undefined;
|
|
90
|
+
});
|
|
91
|
+
return inFlight;
|
|
92
|
+
};
|
|
93
|
+
const ensureTimer = () => {
|
|
94
|
+
if (timer || closed)
|
|
95
|
+
return;
|
|
96
|
+
timer = setInterval(() => void kick(), intervalMs);
|
|
97
|
+
timer.unref();
|
|
98
|
+
};
|
|
99
|
+
return {
|
|
100
|
+
write(chunk) {
|
|
101
|
+
for (const line of chunk.split("\n")) {
|
|
102
|
+
if (line === "")
|
|
103
|
+
continue;
|
|
104
|
+
let level = "info";
|
|
105
|
+
let ms = Date.now();
|
|
106
|
+
try {
|
|
107
|
+
const rec = JSON.parse(line);
|
|
108
|
+
if (typeof rec.level === "string")
|
|
109
|
+
level = rec.level;
|
|
110
|
+
if (typeof rec.time === "string") {
|
|
111
|
+
const t = Date.parse(rec.time);
|
|
112
|
+
if (!Number.isNaN(t))
|
|
113
|
+
ms = t;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
// not JSON: push it as is
|
|
118
|
+
}
|
|
119
|
+
buffer.push({ level, ns: `${BigInt(ms) * 1000000n}`, line });
|
|
120
|
+
if (buffer.length > maxBuffer) {
|
|
121
|
+
buffer.shift();
|
|
122
|
+
dropped++;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
ensureTimer();
|
|
126
|
+
if (buffer.length >= BATCH_SIZE)
|
|
127
|
+
void kick();
|
|
128
|
+
},
|
|
129
|
+
async flush() {
|
|
130
|
+
while (inFlight || buffer.length > 0) {
|
|
131
|
+
if (inFlight)
|
|
132
|
+
await inFlight;
|
|
133
|
+
else
|
|
134
|
+
await kick();
|
|
135
|
+
}
|
|
136
|
+
},
|
|
137
|
+
async close() {
|
|
138
|
+
closed = true;
|
|
139
|
+
if (timer)
|
|
140
|
+
clearInterval(timer);
|
|
141
|
+
timer = undefined;
|
|
142
|
+
await this.flush();
|
|
143
|
+
},
|
|
144
|
+
};
|
|
145
|
+
}
|
package/dist/log.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Logger } from "@capybara-hq/plugin-sdk";
|
|
2
|
+
import { type LokiOptions } from "./log-loki.js";
|
|
2
3
|
/** 日志级别,按严重程度递增。 */
|
|
3
4
|
export declare const LOG_LEVELS: readonly ["debug", "info", "warn", "error"];
|
|
4
5
|
/** 日志级别。 */
|
|
@@ -11,7 +12,7 @@ export interface KernelLogger extends Logger {
|
|
|
11
12
|
setLevel(level: string): void;
|
|
12
13
|
/**
|
|
13
14
|
* 登记部署方(运维)给的密钥值:内核配置与插件部署级配置里的 secret 字段。此后日志消息与字段里出现这些值都替换为
|
|
14
|
-
* `[REDACTED]`(顶层的 `
|
|
15
|
+
* `[REDACTED]`(顶层的 `time` / `level` 不打码)。字符串与数字(转成字符串)登记,布尔不登记;
|
|
15
16
|
* 数组与对象递归展开其中的标量;短于 4 个字符的值不登记(会误伤普通文本)。
|
|
16
17
|
* 数值字段的字符串形式等于已登记值时同样打码。
|
|
17
18
|
*/
|
|
@@ -27,6 +28,8 @@ export interface KernelLogger extends Logger {
|
|
|
27
28
|
* 报错说明)之前用。
|
|
28
29
|
*/
|
|
29
30
|
redact(text: string): string;
|
|
31
|
+
/** Waits until buffered records reached every output (Loki); never rejects. Call before exit. */
|
|
32
|
+
flush(): Promise<void>;
|
|
30
33
|
}
|
|
31
34
|
/**
|
|
32
35
|
* 用与 logger 相同的匹配器打码任意文本(工具结果等不经过日志的内容用):展开规则同 `addOrgSecrets`
|
|
@@ -34,15 +37,43 @@ export interface KernelLogger extends Logger {
|
|
|
34
37
|
* 不得小于匹配器的分桶前缀长度 4)。没有可用的值时原样返回。
|
|
35
38
|
*/
|
|
36
39
|
export declare function createRedactor(values: readonly unknown[], minLength?: number): (s: string) => string;
|
|
37
|
-
/**
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
40
|
+
/** Console format: `pretty` (pino-pretty, one line per record), `json`, or `auto` (pretty on a TTY). */
|
|
41
|
+
export declare const LOG_FORMATS: readonly ["auto", "pretty", "json"];
|
|
42
|
+
/** Console format. */
|
|
43
|
+
export type LogFormat = (typeof LOG_FORMATS)[number];
|
|
44
|
+
/** A minimal writable the console output goes to (`process.stdout` by default). */
|
|
45
|
+
export interface LogOutput {
|
|
46
|
+
write(chunk: string): unknown;
|
|
47
|
+
}
|
|
48
|
+
/** Options of {@link createLogger}. */
|
|
49
|
+
export interface LoggerOptions {
|
|
50
|
+
/**
|
|
51
|
+
* Receives every record as one JSON line (no trailing newline) instead of the console; the
|
|
52
|
+
* console format is ignored. Used by tests and embedders that collect records themselves.
|
|
53
|
+
*/
|
|
45
54
|
write?: (line: string) => void;
|
|
55
|
+
/** Console format; default `auto`. */
|
|
56
|
+
format?: LogFormat;
|
|
57
|
+
/** Console stream; default `process.stdout`. */
|
|
58
|
+
out?: LogOutput;
|
|
59
|
+
/** Whether the console is a terminal (picks `auto` and colors); default `process.stdout.isTTY`. */
|
|
60
|
+
isTTY?: boolean;
|
|
61
|
+
/** Also push every record to Loki. */
|
|
62
|
+
loki?: LokiOptions;
|
|
63
|
+
/** Fields on every record (e.g. `role`); log calls cannot overwrite them. */
|
|
64
|
+
base?: Record<string, string>;
|
|
65
|
+
/** Capacity of the org secret set (tests). */
|
|
46
66
|
orgSecretCapacity?: number;
|
|
67
|
+
/** Called on every matcher rebuild (tests). */
|
|
47
68
|
onMatcherBuild?: () => void;
|
|
48
|
-
}
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Creates the kernel logger on pino. Each record is `{ time, level, msg, ...base, ...fields }` with
|
|
72
|
+
* `level` the label and `time` ISO 8601; fields cannot overwrite time / level / msg or the base
|
|
73
|
+
* fields; an Error becomes name / message / stack (and cause). Redaction happens before pino, so
|
|
74
|
+
* every output sees the same redacted record: deployment secrets (`addSecrets`) in msg and every
|
|
75
|
+
* field string, org secrets (`addOrgSecrets`) in fields only; level and time are never redacted.
|
|
76
|
+
* The console gets pino-pretty or JSON lines per `opts.format`; `opts.write` replaces the console
|
|
77
|
+
* with a JSON line sink.
|
|
78
|
+
*/
|
|
79
|
+
export declare function createLogger(level?: string, opts?: LoggerOptions): KernelLogger;
|