@tinyshed/tinystore 0.1.0-rc.7 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/blobs.d.ts +1 -1
- package/dist/blobs.js +13 -5
- package/dist/cancel.d.ts +16 -2
- package/dist/cancel.js +36 -2
- package/dist/channels.js +20 -6
- package/dist/config.d.ts +49 -1
- package/dist/config.js +86 -7
- package/dist/connection.d.ts +52 -10
- package/dist/connection.js +305 -73
- package/dist/handles.d.ts +10 -1
- package/dist/handles.js +27 -4
- package/dist/index.d.ts +4 -4
- package/dist/jobs.d.ts +33 -1
- package/dist/jobs.js +45 -0
- package/dist/logger.d.ts +22 -10
- package/dist/logger.js +118 -28
- package/dist/metrics.d.ts +19 -0
- package/dist/metrics.js +22 -1
- package/dist/query.js +33 -9
- package/dist/records.d.ts +30 -2
- package/dist/records.js +8 -0
- package/dist/runtime/addon.js +3 -2
- package/dist/runtime/bun.js +33 -12
- package/dist/runtime/node.js +30 -8
- package/dist/runtime/pipe.d.ts +2 -0
- package/dist/runtime/pipe.js +45 -20
- package/dist/runtime/shared.d.ts +28 -4
- package/dist/runtime/shared.js +125 -13
- package/dist/runtime.d.ts +28 -6
- package/dist/session.d.ts +9 -5
- package/dist/session.js +113 -31
- package/dist/sql.d.ts +18 -5
- package/dist/sql.js +44 -11
- package/dist/store.d.ts +25 -11
- package/dist/store.js +67 -9
- package/dist/wire/codec.d.ts +5 -0
- package/dist/wire/codec.js +10 -0
- package/dist/wire/frame.d.ts +2 -0
- package/dist/wire/frame.js +4 -0
- package/dist/wire/msgpack.d.ts +7 -3
- package/dist/wire/msgpack.js +11 -4
- package/dist/wire/protocol.d.ts +246 -36
- package/dist/wire/protocol.js +387 -101
- package/package.json +9 -8
- package/src/blobs.ts +13 -5
- package/src/cancel.ts +44 -2
- package/src/channels.ts +21 -6
- package/src/config.ts +138 -12
- package/src/connection.ts +377 -77
- package/src/handles.ts +32 -4
- package/src/index.ts +5 -0
- package/src/jobs.ts +63 -0
- package/src/logger.ts +158 -40
- package/src/metrics.ts +46 -1
- package/src/query.ts +33 -13
- package/src/records.ts +40 -2
- package/src/runtime/addon.ts +3 -2
- package/src/runtime/bun.ts +44 -9
- package/src/runtime/node.ts +30 -5
- package/src/runtime/pipe.ts +49 -19
- package/src/runtime/shared.ts +137 -16
- package/src/runtime.ts +31 -6
- package/src/session.ts +129 -30
- package/src/sql.ts +48 -14
- package/src/store.ts +81 -14
- package/src/wire/codec.ts +11 -0
- package/src/wire/frame.ts +5 -0
- package/src/wire/msgpack.ts +11 -4
- package/src/wire/protocol.ts +404 -103
package/src/handles.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type Connection, goesAgain, type Idempotence, type Link } from './connection.ts'
|
|
2
2
|
import { InvalidError } from './errors.ts'
|
|
3
|
+
import { ConnectionLostError } from './session.ts'
|
|
3
4
|
import type { Key } from './wire/codec.ts'
|
|
4
5
|
import { Reader, Writer } from './wire/msgpack.ts'
|
|
5
6
|
import { Handle } from './wire/protocol.ts'
|
|
@@ -158,7 +159,12 @@ export function handleOn(connection: Connection, method: number, open: Open): Pr
|
|
|
158
159
|
const key = `${method}:${encoded}`
|
|
159
160
|
let opening = connection.handles.get(key)
|
|
160
161
|
if (opening === undefined) {
|
|
161
|
-
opening = connection.session.call(method, open).then(
|
|
162
|
+
opening = connection.session.call(method, open).then(
|
|
163
|
+
body => Handle.decode(body).handle ?? 0,
|
|
164
|
+
(err: unknown) => {
|
|
165
|
+
throw unsent(err)
|
|
166
|
+
},
|
|
167
|
+
)
|
|
162
168
|
connection.handles.set(key, opening)
|
|
163
169
|
opening.catch(() => connection.handles.delete(key))
|
|
164
170
|
}
|
|
@@ -172,11 +178,33 @@ export function handleOn(connection: Connection, method: number, open: Open): Pr
|
|
|
172
178
|
return opening
|
|
173
179
|
}
|
|
174
180
|
|
|
175
|
-
/**
|
|
181
|
+
/**
|
|
182
|
+
* A connection lost while a handle opened, as the call that needed the handle
|
|
183
|
+
* meets it: the open's REQUEST had left, the call's own had not, so the call
|
|
184
|
+
* goes again on the next connection, where the handle opens again.
|
|
185
|
+
*/
|
|
186
|
+
export function unsent(err: unknown): unknown {
|
|
187
|
+
return err instanceof ConnectionLostError && err.sent
|
|
188
|
+
? new ConnectionLostError(err.message, false)
|
|
189
|
+
: err
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const named = /^[a-z0-9][a-z0-9_-]{0,63}$/
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* A bucket's or a queue's name, as every engine takes it: [a-z0-9][a-z0-9_-]{0,63},
|
|
196
|
+
* with the nearest name that is when there is one: `focusroom.rooms` as `focusroom_rooms`.
|
|
197
|
+
*/
|
|
176
198
|
export function checkName(name: string, of: string): void {
|
|
177
|
-
if (
|
|
199
|
+
if (!named.test(name)) {
|
|
200
|
+
const nearest = name
|
|
201
|
+
.toLowerCase()
|
|
202
|
+
.replace(/[^a-z0-9_-]/g, '_')
|
|
203
|
+
.replace(/^[_-]+/, '')
|
|
204
|
+
.slice(0, 64)
|
|
205
|
+
const offered = named.test(nearest) ? `, as ${JSON.stringify(nearest)}` : ''
|
|
178
206
|
throw new InvalidError(
|
|
179
|
-
`the ${of} name ${JSON.stringify(name)}: a name is [a-z0-9][a-z0-9_-]{0,63}`,
|
|
207
|
+
`the ${of} name ${JSON.stringify(name)}: a name is [a-z0-9][a-z0-9_-]{0,63}${offered}`,
|
|
180
208
|
)
|
|
181
209
|
}
|
|
182
210
|
}
|
package/src/index.ts
CHANGED
|
@@ -33,7 +33,9 @@ export {
|
|
|
33
33
|
type EnvLayer,
|
|
34
34
|
type EnvOptions,
|
|
35
35
|
type Explained,
|
|
36
|
+
type ExplainGiven,
|
|
36
37
|
type ExplainLine,
|
|
38
|
+
type ExplainOrigin,
|
|
37
39
|
env,
|
|
38
40
|
type FileLayer,
|
|
39
41
|
file,
|
|
@@ -56,6 +58,7 @@ export { type ConsoleFormat, type ConsoleTime, secrets } from './console.ts'
|
|
|
56
58
|
export * from './errors.ts'
|
|
57
59
|
export type { Page } from './handles.ts'
|
|
58
60
|
export type {
|
|
61
|
+
Adding,
|
|
59
62
|
AddOptions,
|
|
60
63
|
Answer,
|
|
61
64
|
Backoff,
|
|
@@ -127,6 +130,7 @@ export {
|
|
|
127
130
|
type Percentile,
|
|
128
131
|
type PercentileBucket,
|
|
129
132
|
type Plan,
|
|
133
|
+
type PrometheusOptions,
|
|
130
134
|
type Query as MetricsQuery,
|
|
131
135
|
type Series,
|
|
132
136
|
type SeriesInput,
|
|
@@ -151,6 +155,7 @@ export type {
|
|
|
151
155
|
EventsOptions,
|
|
152
156
|
Fields,
|
|
153
157
|
Level,
|
|
158
|
+
Levels,
|
|
154
159
|
Lines,
|
|
155
160
|
NewEvent,
|
|
156
161
|
Page as EventsPage,
|
package/src/jobs.ts
CHANGED
|
@@ -513,6 +513,7 @@ async function* watchJob<T>(source: Source<T>, id: string): AsyncGenerator<Job<T
|
|
|
513
513
|
let last: Read<typeof JobsJob.fields> | undefined
|
|
514
514
|
for (;;) {
|
|
515
515
|
const stream = await source.link.run('read', connection => openWatch(connection, source, id))
|
|
516
|
+
const unfollow = source.link.follow(stream)
|
|
516
517
|
try {
|
|
517
518
|
for (;;) {
|
|
518
519
|
const event = await stream.next()
|
|
@@ -532,6 +533,7 @@ async function* watchJob<T>(source: Source<T>, id: string): AsyncGenerator<Job<T
|
|
|
532
533
|
}
|
|
533
534
|
} finally {
|
|
534
535
|
stream.cancel()
|
|
536
|
+
unfollow()
|
|
535
537
|
}
|
|
536
538
|
}
|
|
537
539
|
}
|
|
@@ -602,6 +604,21 @@ export class Queue<T> {
|
|
|
602
604
|
return (await this.#changed(methods['jobs.add'], fields)) === true
|
|
603
605
|
}
|
|
604
606
|
|
|
607
|
+
/**
|
|
608
|
+
* A job's add for a batch of the database the queue was opened from:
|
|
609
|
+
* `db.batch` writes it with the batch's statements, so the job commits
|
|
610
|
+
* with the rows or not at all, and shares the commit with every other
|
|
611
|
+
* write of the file, where `db.tx` holds the writer and pays a sync of
|
|
612
|
+
* its own. It adds as `add` does, unless its id is taken: its `Done` in
|
|
613
|
+
* the batch's answer changed 1 job, or 0.
|
|
614
|
+
*
|
|
615
|
+
* await db.batch([sql`update notes set body = ${body} where id = ${id}`, indexing.adding({ noteId: id })])
|
|
616
|
+
*/
|
|
617
|
+
adding(value: T, options: AddOptions = {}): Adding {
|
|
618
|
+
const call = { ...jobFields(options), id: options.id, value: this.#source.values.encode(value) }
|
|
619
|
+
return new Adding({ name: this.name, source: this.#source as Source<unknown>, call })
|
|
620
|
+
}
|
|
621
|
+
|
|
605
622
|
/**
|
|
606
623
|
* Makes the id's job this value at this time, whatever it was: one waiting
|
|
607
624
|
* is made new, a failed one starts again, and a running one runs once more
|
|
@@ -767,6 +784,52 @@ export class TxQueue<T> {
|
|
|
767
784
|
}
|
|
768
785
|
}
|
|
769
786
|
|
|
787
|
+
/** A queue's add as a database's batch carries it: the queue, and the add's call but its handle. */
|
|
788
|
+
interface AddingParts {
|
|
789
|
+
name: string
|
|
790
|
+
source: Source<unknown>
|
|
791
|
+
call: Omit<Written<typeof JobsCall.fields>, 'handle'>
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
// reads an add's parts, which only this module sees
|
|
795
|
+
let partsOf: (adding: Adding) => AddingParts
|
|
796
|
+
|
|
797
|
+
/** A job's add that a database's batch carries, from `queue.adding`: it commits with the batch's statements, or not at all. */
|
|
798
|
+
export class Adding {
|
|
799
|
+
readonly #parts: AddingParts
|
|
800
|
+
|
|
801
|
+
/** Comes from `queue.adding`. */
|
|
802
|
+
constructor(parts: AddingParts) {
|
|
803
|
+
this.#parts = parts
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
static {
|
|
807
|
+
partsOf = adding => adding.#parts
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
/**
|
|
812
|
+
* What a batch of `home` sends of an add, on the connection it goes on: the
|
|
813
|
+
* add's call, its queue's handle opened there first. A queue kept outside
|
|
814
|
+
* `home` is refused, since its job would not commit with the rows.
|
|
815
|
+
*/
|
|
816
|
+
export function addingIn(
|
|
817
|
+
adding: Adding,
|
|
818
|
+
home: Home,
|
|
819
|
+
): (connection: Connection) => Promise<{ method: number; body: Uint8Array }> {
|
|
820
|
+
const { name, source, call } = partsOf(adding)
|
|
821
|
+
if (source.home !== home) {
|
|
822
|
+
const kept = source.home === undefined ? 'jobs.db' : source.home.describe
|
|
823
|
+
throw new InvalidError(
|
|
824
|
+
`${home.describe}: jobs queue ${name}: kept in ${kept}, outside this batch of ${home.describe}`,
|
|
825
|
+
)
|
|
826
|
+
}
|
|
827
|
+
return async connection => {
|
|
828
|
+
const handle = await handleOn(connection, source.openMethod, source.open)
|
|
829
|
+
return { method: methods['jobs.add'], body: JobsCall.encode({ ...call, handle }) }
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
|
|
770
833
|
/** A write the store's transaction keeps for its commit: its method, and its call or the id it cancels. */
|
|
771
834
|
export interface JobOp {
|
|
772
835
|
source: Source<unknown>
|
package/src/logger.ts
CHANGED
|
@@ -26,13 +26,13 @@ import {
|
|
|
26
26
|
} from './console.ts'
|
|
27
27
|
import { parseDotenv } from './dotenv.ts'
|
|
28
28
|
import { InvalidError } from './errors.ts'
|
|
29
|
-
import type { Fields, Level } from './records.ts'
|
|
29
|
+
import type { Fields, Level, Levels } from './records.ts'
|
|
30
30
|
import { currentTrace, type Trace } from './trace.ts'
|
|
31
31
|
|
|
32
32
|
export interface LoggerOptions {
|
|
33
33
|
/** the lines held between writes; past it a line is dropped and counted: 1024 */
|
|
34
34
|
buffer?: number
|
|
35
|
-
/** the least level a line is
|
|
35
|
+
/** the least level a line is printed at, every level when absent */
|
|
36
36
|
level?: Level
|
|
37
37
|
/** how a line is written as it is logged: pretty on a terminal and JSON otherwise when absent */
|
|
38
38
|
console?: ConsoleFormat
|
|
@@ -51,12 +51,16 @@ export interface LoggerOptions {
|
|
|
51
51
|
redact?: readonly string[]
|
|
52
52
|
/** leaves a URL's password in a value as it is, which is otherwise hidden */
|
|
53
53
|
keepUrlPasswords?: boolean
|
|
54
|
-
/**
|
|
54
|
+
/**
|
|
55
|
+
* changes each field's value before it is hidden, kept and shown, and
|
|
56
|
+
* undefined leaves the field out: return the value it was given for a
|
|
57
|
+
* field it leaves as it is
|
|
58
|
+
*/
|
|
55
59
|
replace?: (key: string, value: unknown) => unknown
|
|
56
60
|
/**
|
|
57
|
-
* where LOG_LEVEL, LOG_FORMAT and LOG_TIME are read,
|
|
58
|
-
*
|
|
59
|
-
* reads nothing; the bare names when absent
|
|
61
|
+
* where LOG_LEVEL, LOG_CONSOLE_LEVEL, LOG_FORMAT and LOG_TIME are read,
|
|
62
|
+
* each of which wins over its option: `env('APP')` reads APP_LOG_LEVEL
|
|
63
|
+
* and the rest, and false reads nothing; the bare names when absent
|
|
60
64
|
*/
|
|
61
65
|
env?: EnvLayer | false
|
|
62
66
|
/** adds where each line was logged: source={"function":…,"file":…,"line":…} */
|
|
@@ -82,7 +86,10 @@ interface Shared {
|
|
|
82
86
|
append: ((lines: ConsoleLine[]) => Promise<void>) | undefined
|
|
83
87
|
stream: string
|
|
84
88
|
most: number
|
|
85
|
-
least
|
|
89
|
+
/** the least level kept, which the config log changes while the logger runs */
|
|
90
|
+
kept: number
|
|
91
|
+
/** the least level printed, LOG_CONSOLE_LEVEL's or level's console part; undefined prints what is kept */
|
|
92
|
+
printed: number | undefined
|
|
86
93
|
redact: Redactor | undefined
|
|
87
94
|
replace: ((key: string, value: unknown) => unknown) | undefined
|
|
88
95
|
source: boolean
|
|
@@ -136,14 +143,41 @@ type LogDeclaration = {
|
|
|
136
143
|
time: Setting<string>
|
|
137
144
|
}
|
|
138
145
|
|
|
146
|
+
/** What a logger is told: a logger's options, a store's logger's level in its two parts. */
|
|
147
|
+
type Told = Omit<LoggerOptions, 'level'> & { level?: Level | Levels | undefined }
|
|
148
|
+
|
|
149
|
+
/** A logger's options as the environment leaves them: its level read apart, the least kept and the least printed. */
|
|
150
|
+
type Read = Omit<LoggerOptions, 'level'> & { kept: Level | undefined; printed: Level | undefined }
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* A level's two parts, the least kept and the least printed: one level is
|
|
154
|
+
* the kept alone, which the console then follows.
|
|
155
|
+
*/
|
|
156
|
+
function partsOf(
|
|
157
|
+
level: Level | Levels | undefined,
|
|
158
|
+
): [kept: Level | undefined, printed: Level | undefined] {
|
|
159
|
+
if (typeof level !== 'object') {
|
|
160
|
+
return [level, undefined]
|
|
161
|
+
}
|
|
162
|
+
for (const part of Object.keys(level)) {
|
|
163
|
+
if (part !== 'store' && part !== 'console') {
|
|
164
|
+
throw new InvalidError(
|
|
165
|
+
`a logger's level of ${JSON.stringify(part)}: its parts are store and console`,
|
|
166
|
+
)
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return [level.store, level.console]
|
|
170
|
+
}
|
|
171
|
+
|
|
139
172
|
/** A store's logger of one stream, and what it declares its config `log` with. */
|
|
140
173
|
export function newStoreLogger(
|
|
141
174
|
append: (lines: ConsoleLine[]) => Promise<void>,
|
|
142
175
|
stream: string,
|
|
143
|
-
options:
|
|
176
|
+
options: Told = {},
|
|
144
177
|
): [Logger, LogConfig] {
|
|
145
178
|
const { shared, spelled } = sharedOf(append, stream, options, undefined)
|
|
146
|
-
const
|
|
179
|
+
const [kept] = partsOf(options.level)
|
|
180
|
+
const level = kept === undefined ? 'trace' : nameOf(levelOf(kept))
|
|
147
181
|
const format = options.console ?? (prettyUntold(options, undefined) ? 'pretty' : 'json')
|
|
148
182
|
const time = options.time ?? 'clock'
|
|
149
183
|
const others = (all: readonly string[], one: string) => all.filter(name => name !== one)
|
|
@@ -153,9 +187,10 @@ export function newStoreLogger(
|
|
|
153
187
|
time: oneOf(time, others(times, time)),
|
|
154
188
|
}
|
|
155
189
|
const env = options.env ?? environmentOf('')
|
|
190
|
+
// the level kept, and printed too while the level has no console part
|
|
156
191
|
const take = (name: string) => {
|
|
157
192
|
if (Object.hasOwn(levels, name)) {
|
|
158
|
-
shared.
|
|
193
|
+
shared.kept = levels[name as keyof typeof levels]
|
|
159
194
|
}
|
|
160
195
|
}
|
|
161
196
|
if (env === false) {
|
|
@@ -179,7 +214,7 @@ function nameOf(least: number): string {
|
|
|
179
214
|
function sharedOf(
|
|
180
215
|
append: ((lines: ConsoleLine[]) => Promise<void>) | undefined,
|
|
181
216
|
stream: string,
|
|
182
|
-
options:
|
|
217
|
+
options: Told,
|
|
183
218
|
out: (ConsoleOut & { utc?: boolean }) | undefined,
|
|
184
219
|
): { shared: Shared; spelled: Record<string, string> } {
|
|
185
220
|
if (stream === '') {
|
|
@@ -194,7 +229,8 @@ function sharedOf(
|
|
|
194
229
|
append,
|
|
195
230
|
stream,
|
|
196
231
|
most,
|
|
197
|
-
|
|
232
|
+
kept: read.kept === undefined ? Number.NEGATIVE_INFINITY : levelOf(read.kept),
|
|
233
|
+
printed: read.printed === undefined ? undefined : levelOf(read.printed),
|
|
198
234
|
redact: redactor(read.redact, read.keepUrlPasswords),
|
|
199
235
|
replace: read.replace,
|
|
200
236
|
source: read.source === true,
|
|
@@ -213,14 +249,16 @@ function sharedOf(
|
|
|
213
249
|
type Unread = [name: string, value: string, expected: string]
|
|
214
250
|
|
|
215
251
|
/**
|
|
216
|
-
* LOG_LEVEL, LOG_FORMAT and LOG_TIME over the options,
|
|
217
|
-
* names, as Go and Python read them;
|
|
218
|
-
*
|
|
252
|
+
* LOG_LEVEL, LOG_CONSOLE_LEVEL, LOG_FORMAT and LOG_TIME over the options,
|
|
253
|
+
* each over its own, after the prefix env names, as Go and Python read them;
|
|
254
|
+
* a console turned off stays off. Each variable read is spelled as the
|
|
255
|
+
* config `log` reads it, by its name, but LOG_CONSOLE_LEVEL, which no
|
|
256
|
+
* setting of the config holds.
|
|
219
257
|
*/
|
|
220
|
-
function fromEnvironment(
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
const read = { ...
|
|
258
|
+
function fromEnvironment(options: Told): [Read, Unread[], Record<string, string>] {
|
|
259
|
+
const { level: given, ...rest } = options
|
|
260
|
+
const [kept, printed] = partsOf(given)
|
|
261
|
+
const read: Read = { ...rest, kept, printed }
|
|
224
262
|
const ignored: Unread[] = []
|
|
225
263
|
const spelled: Record<string, string> = {}
|
|
226
264
|
const env = options.env ?? environmentOf('')
|
|
@@ -229,15 +267,14 @@ function fromEnvironment(
|
|
|
229
267
|
}
|
|
230
268
|
const variables = environment(env)
|
|
231
269
|
const name = (variable: string) => (env.prefix === '' ? variable : `${env.prefix}_${variable}`)
|
|
232
|
-
const level = variables
|
|
270
|
+
const level = levelIn(variables, name('LOG_LEVEL'), ignored)
|
|
233
271
|
if (level !== undefined) {
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
}
|
|
272
|
+
read.kept = levels[level]
|
|
273
|
+
spelled[name('LOG_LEVEL')] = level
|
|
274
|
+
}
|
|
275
|
+
const shown = levelIn(variables, name('LOG_CONSOLE_LEVEL'), ignored)
|
|
276
|
+
if (shown !== undefined) {
|
|
277
|
+
read.printed = levels[shown]
|
|
241
278
|
}
|
|
242
279
|
const format = variables(name('LOG_FORMAT'))
|
|
243
280
|
if (format !== undefined) {
|
|
@@ -262,6 +299,24 @@ function fromEnvironment(
|
|
|
262
299
|
return [read, ignored, spelled]
|
|
263
300
|
}
|
|
264
301
|
|
|
302
|
+
/** The level a variable names, the case ignored and warning read as warn; a value it cannot mean is ignored. */
|
|
303
|
+
function levelIn(
|
|
304
|
+
variables: (name: string) => string | undefined,
|
|
305
|
+
name: string,
|
|
306
|
+
ignored: Unread[],
|
|
307
|
+
): keyof typeof levels | undefined {
|
|
308
|
+
const text = variables(name)
|
|
309
|
+
if (text === undefined) {
|
|
310
|
+
return undefined
|
|
311
|
+
}
|
|
312
|
+
const known = text.toLowerCase() === 'warning' ? 'warn' : text.toLowerCase()
|
|
313
|
+
if (Object.hasOwn(levels, known)) {
|
|
314
|
+
return known as keyof typeof levels
|
|
315
|
+
}
|
|
316
|
+
ignored.push([name, text, 'trace, debug, info, warn or error'])
|
|
317
|
+
return undefined
|
|
318
|
+
}
|
|
319
|
+
|
|
265
320
|
/** The variables of env's files, the process's or `from`'s over them, a missing file skipped; an empty one is none. */
|
|
266
321
|
function environment(env: EnvLayer): (name: string) => string | undefined {
|
|
267
322
|
const fromFiles: Record<string, string> = {}
|
|
@@ -314,14 +369,17 @@ function forceColor(): boolean {
|
|
|
314
369
|
}
|
|
315
370
|
|
|
316
371
|
/** Whether a console told no format prints pretty lines: on a terminal, or where FORCE_COLOR asks. */
|
|
317
|
-
function prettyUntold(options: LoggerOptions, out: ConsoleOut | undefined): boolean {
|
|
372
|
+
function prettyUntold(options: Pick<LoggerOptions, 'to'>, out: ConsoleOut | undefined): boolean {
|
|
318
373
|
const stream: ConsoleOut = out ?? options.to ?? process.stderr
|
|
319
374
|
// FORCE_COLOR other than 0 says a person reads a pipe, as an IDE's run console is one
|
|
320
375
|
return stream.isTTY === true || (out === undefined && forceColor())
|
|
321
376
|
}
|
|
322
377
|
|
|
323
378
|
/** How a logger prints its lines, or undefined when it prints none. */
|
|
324
|
-
function printer(
|
|
379
|
+
function printer(
|
|
380
|
+
options: Pick<LoggerOptions, 'console' | 'to' | 'time' | 'hideStream'>,
|
|
381
|
+
out?: ConsoleOut & { utc?: boolean },
|
|
382
|
+
): Shared['print'] {
|
|
325
383
|
if (options.console === 'off') {
|
|
326
384
|
return undefined
|
|
327
385
|
}
|
|
@@ -436,12 +494,14 @@ export class Logger {
|
|
|
436
494
|
}
|
|
437
495
|
|
|
438
496
|
#log(level: number, message: string, attrs: Fields | undefined): void {
|
|
439
|
-
|
|
497
|
+
// a line that no place of the logger takes is never encoded
|
|
498
|
+
if (!prints(this.#shared, level) && !keeps(this.#shared, level)) {
|
|
440
499
|
return
|
|
441
500
|
}
|
|
442
501
|
this.#emit({ level, msg: message }, attrs)
|
|
443
502
|
}
|
|
444
503
|
|
|
504
|
+
/** Prints a line from the level printed and keeps it from the level kept; an event, which has none, both. */
|
|
445
505
|
#emit(what: Pick<ConsoleLine, 'level' | 'event' | 'msg'>, attrs: Fields | undefined): void {
|
|
446
506
|
const trace = traceOf(currentTrace())
|
|
447
507
|
const shared = this.#shared
|
|
@@ -454,8 +514,12 @@ export class Logger {
|
|
|
454
514
|
attrs: shared.source ? [...encoded, sourceKey, sourceOfCaller()] : encoded,
|
|
455
515
|
...trace,
|
|
456
516
|
}
|
|
457
|
-
|
|
458
|
-
|
|
517
|
+
if (what.level === undefined || prints(shared, what.level)) {
|
|
518
|
+
shared.print?.(line)
|
|
519
|
+
}
|
|
520
|
+
if (what.level === undefined || keeps(shared, what.level)) {
|
|
521
|
+
this.#queue(line)
|
|
522
|
+
}
|
|
459
523
|
}
|
|
460
524
|
|
|
461
525
|
#queue(line: ConsoleLine): void {
|
|
@@ -479,6 +543,16 @@ export class Logger {
|
|
|
479
543
|
}
|
|
480
544
|
}
|
|
481
545
|
|
|
546
|
+
/** Whether a logger prints a line of level: it has a console, and the level is the level printed or above. */
|
|
547
|
+
function prints(shared: Shared, level: number): boolean {
|
|
548
|
+
return shared.print !== undefined && level >= (shared.printed ?? shared.kept)
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
/** Whether a logger keeps a line of level: it is a store's, and the level is its level or above. */
|
|
552
|
+
function keeps(shared: Shared, level: number): boolean {
|
|
553
|
+
return shared.append !== undefined && level >= shared.kept
|
|
554
|
+
}
|
|
555
|
+
|
|
482
556
|
/**
|
|
483
557
|
* Writes what the buffer holds, and again while half of it filled during the
|
|
484
558
|
* write before. A full buffer drops a line before the line could ask for a
|
|
@@ -491,21 +565,61 @@ async function writeWaiting(
|
|
|
491
565
|
): Promise<void> {
|
|
492
566
|
let batch = shared.waiting.splice(0)
|
|
493
567
|
while (batch.length > 0) {
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
568
|
+
for (const piece of pieces(batch)) {
|
|
569
|
+
try {
|
|
570
|
+
await append(piece)
|
|
571
|
+
shared.failures?.succeeded()
|
|
572
|
+
} catch (err) {
|
|
573
|
+
shared.dropped += piece.length
|
|
574
|
+
shared.failures?.failed(err)
|
|
575
|
+
}
|
|
500
576
|
}
|
|
501
577
|
batch = shared.waiting.length >= shared.most / 2 ? shared.waiting.splice(0) : []
|
|
502
578
|
}
|
|
503
579
|
}
|
|
504
580
|
|
|
581
|
+
/**
|
|
582
|
+
* The text a write carries at most. A request's body holds 2 MiB, and a
|
|
583
|
+
* buffer of long lines, or one made larger, held more: the whole write was
|
|
584
|
+
* refused and every line in it dropped. Counted in UTF-16 units, three bytes
|
|
585
|
+
* each at worst, so a piece stays within the body however its text is spelt.
|
|
586
|
+
*/
|
|
587
|
+
const PIECE = 512 * 1024
|
|
588
|
+
|
|
589
|
+
/** The lines in writes of at most PIECE of text each, in their order. */
|
|
590
|
+
function pieces(lines: ConsoleLine[]): ConsoleLine[][] {
|
|
591
|
+
const out: ConsoleLine[][] = []
|
|
592
|
+
let piece: ConsoleLine[] = []
|
|
593
|
+
let size = 0
|
|
594
|
+
for (const line of lines) {
|
|
595
|
+
const weight = textOf(line)
|
|
596
|
+
if (piece.length > 0 && size + weight > PIECE) {
|
|
597
|
+
out.push(piece)
|
|
598
|
+
piece = []
|
|
599
|
+
size = 0
|
|
600
|
+
}
|
|
601
|
+
piece.push(line)
|
|
602
|
+
size += weight
|
|
603
|
+
}
|
|
604
|
+
if (piece.length > 0) {
|
|
605
|
+
out.push(piece)
|
|
606
|
+
}
|
|
607
|
+
return out
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
/** A line's text, with a little for its time, level and framing. */
|
|
611
|
+
function textOf(line: ConsoleLine): number {
|
|
612
|
+
let size = 64 + (line.msg?.length ?? 0) + (line.event?.length ?? 0)
|
|
613
|
+
for (const part of line.attrs) size += part.length
|
|
614
|
+
for (const part of line.context) size += part.length
|
|
615
|
+
return size
|
|
616
|
+
}
|
|
617
|
+
|
|
505
618
|
/**
|
|
506
619
|
* Fields as key, JSON pairs, as a console writes them and the store keeps
|
|
507
|
-
* them:
|
|
508
|
-
*
|
|
620
|
+
* them: a field whose value is undefined left out, as JSON.stringify leaves
|
|
621
|
+
* it out of an object, an Error's value its stack, which JSON would write as
|
|
622
|
+
* {}, a secret's value hidden at any depth, and a URL's password unless kept.
|
|
509
623
|
*/
|
|
510
624
|
export function encodeFields(
|
|
511
625
|
fields: Fields,
|
|
@@ -518,6 +632,10 @@ export function encodeFields(
|
|
|
518
632
|
const flat: string[] = []
|
|
519
633
|
for (const [key, given] of pairs) {
|
|
520
634
|
const value = replace === undefined ? given : replace(key, given)
|
|
635
|
+
// before it is hidden, as Rust's replace drops a field: a secret given no value is no field
|
|
636
|
+
if (value === undefined) {
|
|
637
|
+
continue
|
|
638
|
+
}
|
|
521
639
|
const shown = value instanceof Error ? (value.stack ?? String(value)) : value
|
|
522
640
|
if (redact?.hides(key)) {
|
|
523
641
|
flat.push(key, redacted)
|
package/src/metrics.ts
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
MetricsDamages,
|
|
29
29
|
MetricsDeleted,
|
|
30
30
|
MetricsDescription,
|
|
31
|
+
MetricsExposition,
|
|
31
32
|
MetricsIngest,
|
|
32
33
|
MetricsLatest,
|
|
33
34
|
MetricsNameAsked,
|
|
@@ -102,6 +103,16 @@ export interface SeriesInput {
|
|
|
102
103
|
keep?: Duration | undefined
|
|
103
104
|
}
|
|
104
105
|
|
|
106
|
+
/** What an ingest of an exporter's text gives every series of it. */
|
|
107
|
+
export interface PrometheusOptions {
|
|
108
|
+
/** labels every series gets beside its own, such as the instance scraped; one the text gives too is refused, naming its line */
|
|
109
|
+
labels?: Record<string, LabelValue> | undefined
|
|
110
|
+
/** the time of every sample the text gives none; the store's clock unless given */
|
|
111
|
+
at?: Time | undefined
|
|
112
|
+
/** how long every name of the text keeps its samples */
|
|
113
|
+
keep?: Duration | undefined
|
|
114
|
+
}
|
|
115
|
+
|
|
105
116
|
/** A series as a read gives it: its samples as two columns. */
|
|
106
117
|
export interface Series {
|
|
107
118
|
name: string
|
|
@@ -206,7 +217,14 @@ export interface Plan {
|
|
|
206
217
|
series: number
|
|
207
218
|
bytes: number
|
|
208
219
|
decoded: number
|
|
209
|
-
exceeds:
|
|
220
|
+
exceeds:
|
|
221
|
+
| {
|
|
222
|
+
/** series, bytes read, samples decoded or answer */
|
|
223
|
+
limit: string
|
|
224
|
+
wanted: number
|
|
225
|
+
bound: number
|
|
226
|
+
}
|
|
227
|
+
| undefined
|
|
210
228
|
}
|
|
211
229
|
|
|
212
230
|
/** A page's options: a prefix, how many, and where the page before ended. */
|
|
@@ -285,6 +303,33 @@ export class Metrics {
|
|
|
285
303
|
)
|
|
286
304
|
}
|
|
287
305
|
|
|
306
|
+
/**
|
|
307
|
+
* Reads the text an exporter serves, in Prometheus' format or OpenMetrics', and writes its
|
|
308
|
+
* series, every one or none: a counter as a counter, a histogram's _bucket series as
|
|
309
|
+
* histogram. A line it cannot read is refused, naming the line.
|
|
310
|
+
*
|
|
311
|
+
* const text = await (await fetch('http://web-1:9100/metrics')).text()
|
|
312
|
+
* await store.metrics.ingestPrometheus(text, { labels: { instance: 'web-1' } })
|
|
313
|
+
*/
|
|
314
|
+
async ingestPrometheus(
|
|
315
|
+
text: string | Uint8Array,
|
|
316
|
+
options: PrometheusOptions = {},
|
|
317
|
+
): Promise<void> {
|
|
318
|
+
const labels: Labels = {}
|
|
319
|
+
for (const [label, value] of Object.entries(options.labels ?? {})) {
|
|
320
|
+
labels[label] = String(value)
|
|
321
|
+
}
|
|
322
|
+
const body = MetricsExposition.encode({
|
|
323
|
+
text: typeof text === 'string' ? new TextEncoder().encode(text) : text,
|
|
324
|
+
labels,
|
|
325
|
+
at: options.at === undefined ? undefined : unixMs(options.at),
|
|
326
|
+
keep: options.keep === undefined ? undefined : ms(options.keep),
|
|
327
|
+
})
|
|
328
|
+
await this.#link.run('write', connection =>
|
|
329
|
+
connection.session.call(methods['metrics.ingestPrometheus'], body),
|
|
330
|
+
)
|
|
331
|
+
}
|
|
332
|
+
|
|
288
333
|
/** The samples of each series the query finds, in its range: whole, or a limit error naming its bound. */
|
|
289
334
|
async read(query: Query): Promise<Series[]> {
|
|
290
335
|
const parts = await this.#download('metrics.read', MetricsQuery.encode(queryOf(query)))
|
package/src/query.ts
CHANGED
|
@@ -868,18 +868,31 @@ function render(parts: Parts): Sql {
|
|
|
868
868
|
return new Sql(text, values)
|
|
869
869
|
}
|
|
870
870
|
|
|
871
|
-
/**
|
|
871
|
+
/**
|
|
872
|
+
* The condition that starts a page after a row's order values. SQLite orders
|
|
873
|
+
* a null before every value, first in ascending order and last in descending,
|
|
874
|
+
* and a comparison with a null is never true: equal is `is`, and past a null
|
|
875
|
+
* is every value ascending and nothing descending.
|
|
876
|
+
*/
|
|
872
877
|
function keyset(order: Parts['order'], after: readonly SqlValue[]): Sql {
|
|
873
878
|
const alternatives: string[] = []
|
|
874
879
|
const values: SqlArg[] = []
|
|
875
880
|
order.forEach((each, at) => {
|
|
876
|
-
const
|
|
881
|
+
const column = each.piece.text
|
|
882
|
+
const value = after[at] ?? null
|
|
883
|
+
if (value === null && each.direction === 'desc') {
|
|
884
|
+
return
|
|
885
|
+
}
|
|
886
|
+
const equal = order.slice(0, at).map(earlier => `${earlier.piece.text} is ?`)
|
|
877
887
|
values.push(...after.slice(0, at))
|
|
878
|
-
|
|
879
|
-
|
|
888
|
+
let past = `${column} is not null`
|
|
889
|
+
if (value !== null) {
|
|
890
|
+
past = each.direction === 'asc' ? `${column} > ?` : `(${column} < ? or ${column} is null)`
|
|
891
|
+
values.push(value)
|
|
892
|
+
}
|
|
880
893
|
alternatives.push(equal.length === 0 ? past : `(${[...equal, past].join(' and ')})`)
|
|
881
894
|
})
|
|
882
|
-
return new Sql(`(${alternatives.join(' or ')})`, values)
|
|
895
|
+
return new Sql(alternatives.length === 0 ? '0' : `(${alternatives.join(' or ')})`, values)
|
|
883
896
|
}
|
|
884
897
|
|
|
885
898
|
/** A short digest of a query, which a cursor carries so that another query refuses it. */
|
|
@@ -893,10 +906,14 @@ function digestOf(statement: Sql): string {
|
|
|
893
906
|
return hash.toString(36)
|
|
894
907
|
}
|
|
895
908
|
|
|
909
|
+
/** A cursor of order values: a bigint, and an infinity JSON has no number for, as their text. */
|
|
896
910
|
function cursorText(values: readonly SqlValue[], digest: string): string {
|
|
897
|
-
const json = JSON.stringify({ q: digest, v: values }, (_, v) =>
|
|
898
|
-
typeof v === 'bigint'
|
|
899
|
-
|
|
911
|
+
const json = JSON.stringify({ q: digest, v: values }, (_, v) => {
|
|
912
|
+
if (typeof v === 'bigint') {
|
|
913
|
+
return { n: String(v) }
|
|
914
|
+
}
|
|
915
|
+
return typeof v === 'number' && !Number.isFinite(v) ? { r: String(v) } : v
|
|
916
|
+
})
|
|
900
917
|
return Buffer.from(json).toString('base64url')
|
|
901
918
|
}
|
|
902
919
|
|
|
@@ -912,11 +929,14 @@ function cursorOf(cursor: string, digest: string): SqlValue[] {
|
|
|
912
929
|
"a cursor of another query: a page's next reads only the query that made it",
|
|
913
930
|
)
|
|
914
931
|
}
|
|
915
|
-
return read.v.map(v =>
|
|
916
|
-
typeof v === 'object' && v !== null && 'n' in v
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
932
|
+
return read.v.map(v => {
|
|
933
|
+
if (typeof v === 'object' && v !== null && 'n' in v) {
|
|
934
|
+
return BigInt((v as { n: string }).n)
|
|
935
|
+
}
|
|
936
|
+
return typeof v === 'object' && v !== null && 'r' in v
|
|
937
|
+
? Number((v as { r: string }).r)
|
|
938
|
+
: (v as SqlValue)
|
|
939
|
+
})
|
|
920
940
|
}
|
|
921
941
|
|
|
922
942
|
/** An insert of rows of the same columns, the first row's. */
|