@walkeros/server-destination-file 3.4.0-next-1776749829492
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +97 -0
- package/dist/dev.d.mts +148 -0
- package/dist/dev.d.ts +148 -0
- package/dist/dev.js +1 -0
- package/dist/dev.js.map +1 -0
- package/dist/dev.mjs +1 -0
- package/dist/dev.mjs.map +1 -0
- package/dist/examples/index.d.mts +113 -0
- package/dist/examples/index.d.ts +113 -0
- package/dist/examples/index.js +178 -0
- package/dist/examples/index.mjs +156 -0
- package/dist/index.d.mts +78 -0
- package/dist/index.d.ts +78 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +1 -0
- package/dist/index.mjs.map +1 -0
- package/dist/walkerOS.json +508 -0
- package/package.json +76 -0
package/README.md
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# @walkeros/server-destination-file
|
|
2
|
+
|
|
3
|
+
Local file sink for walkerOS server flows. Appends events to a file in JSONL,
|
|
4
|
+
TSV, or CSV format. Useful for debug logging, audit trails, replay sources, and
|
|
5
|
+
lightweight local persistence without standing up a database.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install @walkeros/server-destination-file
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Usage
|
|
14
|
+
|
|
15
|
+
Add to your `flow.json`:
|
|
16
|
+
|
|
17
|
+
```json
|
|
18
|
+
{
|
|
19
|
+
"destinations": {
|
|
20
|
+
"log": {
|
|
21
|
+
"package": "@walkeros/server-destination-file",
|
|
22
|
+
"config": {
|
|
23
|
+
"settings": {
|
|
24
|
+
"filename": "events.jsonl"
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Every event gets appended as a JSON line. The file and its parent directory are
|
|
33
|
+
created on flow startup.
|
|
34
|
+
|
|
35
|
+
## Settings
|
|
36
|
+
|
|
37
|
+
| Field | Type | Default | Notes |
|
|
38
|
+
| ---------- | --------------------------- | -------------------- | ----------------------------------------------------------- |
|
|
39
|
+
| `filename` | `string \| Mapping.Value` | required | Static path or per-event resolution via the mapping DSL. |
|
|
40
|
+
| `format` | `'jsonl' \| 'tsv' \| 'csv'` | `jsonl` | Serialisation format. |
|
|
41
|
+
| `fields` | `string[]` | required for tsv/csv | Event paths used as columns. Object cells JSON-stringified. |
|
|
42
|
+
|
|
43
|
+
## Filename templating
|
|
44
|
+
|
|
45
|
+
`filename` accepts the standard walkerOS `Mapping.Value` shape.
|
|
46
|
+
|
|
47
|
+
### Tenant sharding
|
|
48
|
+
|
|
49
|
+
```json
|
|
50
|
+
{ "filename": { "key": "data.tenant" } }
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### Daily rotation
|
|
54
|
+
|
|
55
|
+
```json
|
|
56
|
+
{
|
|
57
|
+
"filename": {
|
|
58
|
+
"fn": "$code:`events-${new Date(value.timestamp).toISOString().slice(0,10)}.jsonl`"
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Inside the `$code:` function, `value` is the event being processed.
|
|
64
|
+
|
|
65
|
+
## Formats
|
|
66
|
+
|
|
67
|
+
### JSONL (default)
|
|
68
|
+
|
|
69
|
+
One JSON object per line. Ingest with `jq`, `duckdb`, ClickHouse `JSONEachRow`,
|
|
70
|
+
BigQuery external tables, Athena.
|
|
71
|
+
|
|
72
|
+
### TSV / CSV
|
|
73
|
+
|
|
74
|
+
Specify `fields: string[]` listing the event paths to extract as columns. Object
|
|
75
|
+
values are JSON-stringified into a single cell. RFC 4180 quoting for CSV.
|
|
76
|
+
|
|
77
|
+
```json
|
|
78
|
+
{
|
|
79
|
+
"filename": "events.csv",
|
|
80
|
+
"format": "csv",
|
|
81
|
+
"fields": ["timestamp", "name", "data"]
|
|
82
|
+
}
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## Limits
|
|
86
|
+
|
|
87
|
+
- One file handle is opened per resolved filename and kept open until
|
|
88
|
+
`destroy()`. Sharding by high-cardinality keys (e.g. `user.session`) can
|
|
89
|
+
exhaust OS file descriptors.
|
|
90
|
+
- External rotation (e.g. `logrotate`) leaves the cached handle pointing at the
|
|
91
|
+
rotated inode. Use the date-token pattern instead.
|
|
92
|
+
- No batching. Each event is `stream.write`'d individually.
|
|
93
|
+
- Write errors log a warning and drop the event — they never fail the flow.
|
|
94
|
+
|
|
95
|
+
## License
|
|
96
|
+
|
|
97
|
+
MIT
|
package/dist/dev.d.mts
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import * as _walkeros_core_dev from '@walkeros/core/dev';
|
|
2
|
+
import { z } from '@walkeros/core/dev';
|
|
3
|
+
import { DestinationServer } from '@walkeros/server-core';
|
|
4
|
+
import { WriteStream } from 'node:fs';
|
|
5
|
+
import { Flow } from '@walkeros/core';
|
|
6
|
+
|
|
7
|
+
declare const SettingsSchema: z.ZodObject<{
|
|
8
|
+
filename: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>]>;
|
|
9
|
+
format: z.ZodOptional<z.ZodEnum<{
|
|
10
|
+
jsonl: "jsonl";
|
|
11
|
+
tsv: "tsv";
|
|
12
|
+
csv: "csv";
|
|
13
|
+
}>>;
|
|
14
|
+
fields: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
15
|
+
}, z.core.$strip>;
|
|
16
|
+
type Settings = z.infer<typeof SettingsSchema>;
|
|
17
|
+
|
|
18
|
+
declare const MappingSchema: z.ZodObject<{}, z.core.$strip>;
|
|
19
|
+
type Mapping = z.infer<typeof MappingSchema>;
|
|
20
|
+
|
|
21
|
+
declare const settings: _walkeros_core_dev.JSONSchema;
|
|
22
|
+
declare const mapping: _walkeros_core_dev.JSONSchema;
|
|
23
|
+
|
|
24
|
+
type index$1_Mapping = Mapping;
|
|
25
|
+
declare const index$1_MappingSchema: typeof MappingSchema;
|
|
26
|
+
type index$1_Settings = Settings;
|
|
27
|
+
declare const index$1_SettingsSchema: typeof SettingsSchema;
|
|
28
|
+
declare const index$1_mapping: typeof mapping;
|
|
29
|
+
declare const index$1_settings: typeof settings;
|
|
30
|
+
declare namespace index$1 {
|
|
31
|
+
export { type index$1_Mapping as Mapping, index$1_MappingSchema as MappingSchema, type index$1_Settings as Settings, index$1_SettingsSchema as SettingsSchema, index$1_mapping as mapping, index$1_settings as settings };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
type Format = 'jsonl' | 'tsv' | 'csv';
|
|
35
|
+
/**
|
|
36
|
+
* Minimal write-stream interface used by the destination. Matches the
|
|
37
|
+
* subset of node:fs WriteStream needed for append-only writes. Tests
|
|
38
|
+
* inject a fake implementation via env.fs.
|
|
39
|
+
*/
|
|
40
|
+
interface FileWriteStream {
|
|
41
|
+
write: (chunk: string) => boolean;
|
|
42
|
+
end: () => void;
|
|
43
|
+
}
|
|
44
|
+
interface Env extends DestinationServer.Env {
|
|
45
|
+
/**
|
|
46
|
+
* Override the file system primitives. Tests inject a fake here so
|
|
47
|
+
* disk writes are captured in memory.
|
|
48
|
+
*/
|
|
49
|
+
fs?: {
|
|
50
|
+
createWriteStream: (path: string, options: {
|
|
51
|
+
flags: string;
|
|
52
|
+
}) => FileWriteStream | WriteStream;
|
|
53
|
+
mkdir: (path: string, options: {
|
|
54
|
+
recursive: boolean;
|
|
55
|
+
}) => Promise<void>;
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Captured file state for assertions in tests.
|
|
61
|
+
*/
|
|
62
|
+
interface CapturedFile {
|
|
63
|
+
filename: string;
|
|
64
|
+
lines: string[];
|
|
65
|
+
ended: boolean;
|
|
66
|
+
}
|
|
67
|
+
interface SpyState {
|
|
68
|
+
captured: Map<string, CapturedFile>;
|
|
69
|
+
mkdirCalls: string[];
|
|
70
|
+
}
|
|
71
|
+
interface SpyEnv extends Env {
|
|
72
|
+
_spy: SpyState;
|
|
73
|
+
}
|
|
74
|
+
declare const init: SpyEnv;
|
|
75
|
+
declare const push: SpyEnv;
|
|
76
|
+
|
|
77
|
+
type env_CapturedFile = CapturedFile;
|
|
78
|
+
type env_SpyEnv = SpyEnv;
|
|
79
|
+
type env_SpyState = SpyState;
|
|
80
|
+
declare const env_init: typeof init;
|
|
81
|
+
declare const env_push: typeof push;
|
|
82
|
+
declare namespace env {
|
|
83
|
+
export { type env_CapturedFile as CapturedFile, type env_SpyEnv as SpyEnv, type env_SpyState as SpyState, env_init as init, env_push as push };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Raw filename-config shape accepted in step examples and flow.json.
|
|
88
|
+
* Mirrors the runtime Settings.filename but types `fn` as string so
|
|
89
|
+
* `$code:` tags (resolved at bundle time by walkerOS) are expressible
|
|
90
|
+
* here without casting to a function type.
|
|
91
|
+
*/
|
|
92
|
+
interface FileSettingsJson {
|
|
93
|
+
filename: string | {
|
|
94
|
+
key?: string;
|
|
95
|
+
fn?: string | ((value: unknown) => unknown);
|
|
96
|
+
value?: unknown;
|
|
97
|
+
};
|
|
98
|
+
format?: Format;
|
|
99
|
+
fields?: string[];
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Extended step example that carries destination-level settings for the
|
|
103
|
+
* file destination.
|
|
104
|
+
*
|
|
105
|
+
* `out` is modeled at intent level as a single `['fs.writeFile', path, line]`
|
|
106
|
+
* tuple per event (even though the destination internally uses a persistent
|
|
107
|
+
* createWriteStream + write). This matches the design doc's preference for
|
|
108
|
+
* readable intent-level callable tuples over stream-chain fidelity.
|
|
109
|
+
*
|
|
110
|
+
* Dropped / ignored events produce `out: []`.
|
|
111
|
+
*/
|
|
112
|
+
type FileStepExample = Flow.StepExample & {
|
|
113
|
+
settings: FileSettingsJson;
|
|
114
|
+
};
|
|
115
|
+
/** Default JSONL append. Static filename, all defaults. */
|
|
116
|
+
declare const jsonlDefault: FileStepExample;
|
|
117
|
+
/** Baersch-style TSV log: PHP parity case. */
|
|
118
|
+
declare const tsvBaerschLog: FileStepExample;
|
|
119
|
+
/** Tenant sharding via plain key extraction. */
|
|
120
|
+
declare const jsonlTenantShardKey: FileStepExample;
|
|
121
|
+
/**
|
|
122
|
+
* Daily rotation via a mapping fn. In flow.json this is authored as a
|
|
123
|
+
* `$code:` string; walkerOS compiles it to a function at bundle time. For
|
|
124
|
+
* in-process tests we pass the already-compiled function directly so the
|
|
125
|
+
* example exercises the same code path without needing the bundler.
|
|
126
|
+
*/
|
|
127
|
+
declare const jsonlDailyRotation: FileStepExample;
|
|
128
|
+
/** CSV with an object cell. data is JSON-stringified, properly quoted. */
|
|
129
|
+
declare const csvObjectCell: FileStepExample;
|
|
130
|
+
|
|
131
|
+
type step_FileSettingsJson = FileSettingsJson;
|
|
132
|
+
type step_FileStepExample = FileStepExample;
|
|
133
|
+
declare const step_csvObjectCell: typeof csvObjectCell;
|
|
134
|
+
declare const step_jsonlDailyRotation: typeof jsonlDailyRotation;
|
|
135
|
+
declare const step_jsonlDefault: typeof jsonlDefault;
|
|
136
|
+
declare const step_jsonlTenantShardKey: typeof jsonlTenantShardKey;
|
|
137
|
+
declare const step_tsvBaerschLog: typeof tsvBaerschLog;
|
|
138
|
+
declare namespace step {
|
|
139
|
+
export { type step_FileSettingsJson as FileSettingsJson, type step_FileStepExample as FileStepExample, step_csvObjectCell as csvObjectCell, step_jsonlDailyRotation as jsonlDailyRotation, step_jsonlDefault as jsonlDefault, step_jsonlTenantShardKey as jsonlTenantShardKey, step_tsvBaerschLog as tsvBaerschLog };
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
declare const index_env: typeof env;
|
|
143
|
+
declare const index_step: typeof step;
|
|
144
|
+
declare namespace index {
|
|
145
|
+
export { index_env as env, index_step as step };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export { index as examples, index$1 as schemas };
|
package/dist/dev.d.ts
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import * as _walkeros_core_dev from '@walkeros/core/dev';
|
|
2
|
+
import { z } from '@walkeros/core/dev';
|
|
3
|
+
import { DestinationServer } from '@walkeros/server-core';
|
|
4
|
+
import { WriteStream } from 'node:fs';
|
|
5
|
+
import { Flow } from '@walkeros/core';
|
|
6
|
+
|
|
7
|
+
declare const SettingsSchema: z.ZodObject<{
|
|
8
|
+
filename: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>]>;
|
|
9
|
+
format: z.ZodOptional<z.ZodEnum<{
|
|
10
|
+
jsonl: "jsonl";
|
|
11
|
+
tsv: "tsv";
|
|
12
|
+
csv: "csv";
|
|
13
|
+
}>>;
|
|
14
|
+
fields: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
15
|
+
}, z.core.$strip>;
|
|
16
|
+
type Settings = z.infer<typeof SettingsSchema>;
|
|
17
|
+
|
|
18
|
+
declare const MappingSchema: z.ZodObject<{}, z.core.$strip>;
|
|
19
|
+
type Mapping = z.infer<typeof MappingSchema>;
|
|
20
|
+
|
|
21
|
+
declare const settings: _walkeros_core_dev.JSONSchema;
|
|
22
|
+
declare const mapping: _walkeros_core_dev.JSONSchema;
|
|
23
|
+
|
|
24
|
+
type index$1_Mapping = Mapping;
|
|
25
|
+
declare const index$1_MappingSchema: typeof MappingSchema;
|
|
26
|
+
type index$1_Settings = Settings;
|
|
27
|
+
declare const index$1_SettingsSchema: typeof SettingsSchema;
|
|
28
|
+
declare const index$1_mapping: typeof mapping;
|
|
29
|
+
declare const index$1_settings: typeof settings;
|
|
30
|
+
declare namespace index$1 {
|
|
31
|
+
export { type index$1_Mapping as Mapping, index$1_MappingSchema as MappingSchema, type index$1_Settings as Settings, index$1_SettingsSchema as SettingsSchema, index$1_mapping as mapping, index$1_settings as settings };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
type Format = 'jsonl' | 'tsv' | 'csv';
|
|
35
|
+
/**
|
|
36
|
+
* Minimal write-stream interface used by the destination. Matches the
|
|
37
|
+
* subset of node:fs WriteStream needed for append-only writes. Tests
|
|
38
|
+
* inject a fake implementation via env.fs.
|
|
39
|
+
*/
|
|
40
|
+
interface FileWriteStream {
|
|
41
|
+
write: (chunk: string) => boolean;
|
|
42
|
+
end: () => void;
|
|
43
|
+
}
|
|
44
|
+
interface Env extends DestinationServer.Env {
|
|
45
|
+
/**
|
|
46
|
+
* Override the file system primitives. Tests inject a fake here so
|
|
47
|
+
* disk writes are captured in memory.
|
|
48
|
+
*/
|
|
49
|
+
fs?: {
|
|
50
|
+
createWriteStream: (path: string, options: {
|
|
51
|
+
flags: string;
|
|
52
|
+
}) => FileWriteStream | WriteStream;
|
|
53
|
+
mkdir: (path: string, options: {
|
|
54
|
+
recursive: boolean;
|
|
55
|
+
}) => Promise<void>;
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Captured file state for assertions in tests.
|
|
61
|
+
*/
|
|
62
|
+
interface CapturedFile {
|
|
63
|
+
filename: string;
|
|
64
|
+
lines: string[];
|
|
65
|
+
ended: boolean;
|
|
66
|
+
}
|
|
67
|
+
interface SpyState {
|
|
68
|
+
captured: Map<string, CapturedFile>;
|
|
69
|
+
mkdirCalls: string[];
|
|
70
|
+
}
|
|
71
|
+
interface SpyEnv extends Env {
|
|
72
|
+
_spy: SpyState;
|
|
73
|
+
}
|
|
74
|
+
declare const init: SpyEnv;
|
|
75
|
+
declare const push: SpyEnv;
|
|
76
|
+
|
|
77
|
+
type env_CapturedFile = CapturedFile;
|
|
78
|
+
type env_SpyEnv = SpyEnv;
|
|
79
|
+
type env_SpyState = SpyState;
|
|
80
|
+
declare const env_init: typeof init;
|
|
81
|
+
declare const env_push: typeof push;
|
|
82
|
+
declare namespace env {
|
|
83
|
+
export { type env_CapturedFile as CapturedFile, type env_SpyEnv as SpyEnv, type env_SpyState as SpyState, env_init as init, env_push as push };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Raw filename-config shape accepted in step examples and flow.json.
|
|
88
|
+
* Mirrors the runtime Settings.filename but types `fn` as string so
|
|
89
|
+
* `$code:` tags (resolved at bundle time by walkerOS) are expressible
|
|
90
|
+
* here without casting to a function type.
|
|
91
|
+
*/
|
|
92
|
+
interface FileSettingsJson {
|
|
93
|
+
filename: string | {
|
|
94
|
+
key?: string;
|
|
95
|
+
fn?: string | ((value: unknown) => unknown);
|
|
96
|
+
value?: unknown;
|
|
97
|
+
};
|
|
98
|
+
format?: Format;
|
|
99
|
+
fields?: string[];
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Extended step example that carries destination-level settings for the
|
|
103
|
+
* file destination.
|
|
104
|
+
*
|
|
105
|
+
* `out` is modeled at intent level as a single `['fs.writeFile', path, line]`
|
|
106
|
+
* tuple per event (even though the destination internally uses a persistent
|
|
107
|
+
* createWriteStream + write). This matches the design doc's preference for
|
|
108
|
+
* readable intent-level callable tuples over stream-chain fidelity.
|
|
109
|
+
*
|
|
110
|
+
* Dropped / ignored events produce `out: []`.
|
|
111
|
+
*/
|
|
112
|
+
type FileStepExample = Flow.StepExample & {
|
|
113
|
+
settings: FileSettingsJson;
|
|
114
|
+
};
|
|
115
|
+
/** Default JSONL append. Static filename, all defaults. */
|
|
116
|
+
declare const jsonlDefault: FileStepExample;
|
|
117
|
+
/** Baersch-style TSV log: PHP parity case. */
|
|
118
|
+
declare const tsvBaerschLog: FileStepExample;
|
|
119
|
+
/** Tenant sharding via plain key extraction. */
|
|
120
|
+
declare const jsonlTenantShardKey: FileStepExample;
|
|
121
|
+
/**
|
|
122
|
+
* Daily rotation via a mapping fn. In flow.json this is authored as a
|
|
123
|
+
* `$code:` string; walkerOS compiles it to a function at bundle time. For
|
|
124
|
+
* in-process tests we pass the already-compiled function directly so the
|
|
125
|
+
* example exercises the same code path without needing the bundler.
|
|
126
|
+
*/
|
|
127
|
+
declare const jsonlDailyRotation: FileStepExample;
|
|
128
|
+
/** CSV with an object cell. data is JSON-stringified, properly quoted. */
|
|
129
|
+
declare const csvObjectCell: FileStepExample;
|
|
130
|
+
|
|
131
|
+
type step_FileSettingsJson = FileSettingsJson;
|
|
132
|
+
type step_FileStepExample = FileStepExample;
|
|
133
|
+
declare const step_csvObjectCell: typeof csvObjectCell;
|
|
134
|
+
declare const step_jsonlDailyRotation: typeof jsonlDailyRotation;
|
|
135
|
+
declare const step_jsonlDefault: typeof jsonlDefault;
|
|
136
|
+
declare const step_jsonlTenantShardKey: typeof jsonlTenantShardKey;
|
|
137
|
+
declare const step_tsvBaerschLog: typeof tsvBaerschLog;
|
|
138
|
+
declare namespace step {
|
|
139
|
+
export { type step_FileSettingsJson as FileSettingsJson, type step_FileStepExample as FileStepExample, step_csvObjectCell as csvObjectCell, step_jsonlDailyRotation as jsonlDailyRotation, step_jsonlDefault as jsonlDefault, step_jsonlTenantShardKey as jsonlTenantShardKey, step_tsvBaerschLog as tsvBaerschLog };
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
declare const index_env: typeof env;
|
|
143
|
+
declare const index_step: typeof step;
|
|
144
|
+
declare namespace index {
|
|
145
|
+
export { index_env as env, index_step as step };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export { index as examples, index$1 as schemas };
|
package/dist/dev.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";var e,t=Object.defineProperty,s=Object.getOwnPropertyDescriptor,n=Object.getOwnPropertyNames,a=Object.prototype.hasOwnProperty,r=(e,s)=>{for(var n in s)t(e,n,{get:s[n],enumerable:!0})},o={};r(o,{examples:()=>v,schemas:()=>i}),module.exports=(e=o,((e,r,o,i)=>{if(r&&"object"==typeof r||"function"==typeof r)for(let l of n(r))a.call(e,l)||l===o||t(e,l,{get:()=>r[l],enumerable:!(i=s(r,l))||i.enumerable});return e})(t({},"__esModule",{value:!0}),e));var i={};r(i,{MappingSchema:()=>p,SettingsSchema:()=>m,mapping:()=>u,settings:()=>d});var l=require("@walkeros/core/dev"),c=require("@walkeros/core/dev"),m=c.z.object({filename:c.z.union([c.z.string().describe("Static output filename."),c.z.record(c.z.string(),c.z.unknown()).describe('Mapping.Value resolved per event (e.g. { key: "data.tenant" } or { fn: "$code:..." }).')]).describe('Output filename. Static string or Mapping.Value (e.g. { fn: "$code:..." } for daily rotation, { key: "data.tenant" } for sharding).'),format:c.z.enum(["jsonl","tsv","csv"]).optional().describe("Serialisation format. Defaults to jsonl."),fields:c.z.array(c.z.string()).optional().describe("Event paths used as columns for tsv/csv formats. Object values are JSON-stringified. Required when format is tsv or csv.")}),p=require("@walkeros/core/dev").z.object({}).describe("No per-rule overrides for the file destination."),d=(0,l.zodToSchema)(m),u=(0,l.zodToSchema)(p),v={};r(v,{env:()=>f,step:()=>b});var f={};function g(){const e={captured:new Map,mkdirCalls:[]};return{_spy:e,fs:{createWriteStream:t=>{const s=e.captured.get(t),n=null!=s?s:{filename:t,lines:[],ended:!1};s||e.captured.set(t,n);return{write:e=>(n.lines.push(e),!0),end(){n.ended=!0}}},mkdir:async t=>{e.mkdirCalls.push(t)}}}}r(f,{init:()=>j,push:()=>w});var j=g(),w=g(),b={};r(b,{csvObjectCell:()=>k,jsonlDailyRotation:()=>z,jsonlDefault:()=>y,jsonlTenantShardKey:()=>S,tsvBaerschLog:()=>O});var h=require("@walkeros/core"),y={in:(0,h.getEvent)("page view",{timestamp:17e11}),settings:{filename:"events.jsonl"},out:[["fs.writeFile","events.jsonl","jsonl:event"]]},O={in:(0,h.getEvent)("page view",{timestamp:17e11,data:{title:"Docs"},user:{session:"sess-1"},source:{id:"https://example.com/docs",type:"server",previous_id:"https://example.com/"}}),settings:{filename:"storage/mblog.txt",format:"tsv",fields:["timestamp","user.session","name","source.id","data.title","source.previous_id"]},out:[["fs.writeFile","storage/mblog.txt","1700000000000\tsess-1\tpage view\thttps://example.com/docs\tDocs\thttps://example.com/\n"]]},S={in:(0,h.getEvent)("custom event",{timestamp:17e11,data:{tenant:"acme"}}),settings:{filename:{key:"data.tenant"},format:"jsonl"},out:[["fs.writeFile","acme","jsonl:event"]]},z={in:(0,h.getEvent)("order complete",{timestamp:Date.UTC(2026,3,15,12,34,56),data:{id:"ORD-1"}}),settings:{filename:{fn:e=>{var t;const s=null!=(t=e.timestamp)?t:0;return`events-${new Date(s).toISOString().slice(0,10)}.jsonl`}},format:"jsonl"},out:[["fs.writeFile","events-2026-04-15.jsonl","jsonl:event"]]},k={in:(0,h.getEvent)("page view",{timestamp:17e11,data:{title:'Hello, "World"',count:3}}),settings:{filename:"events.csv",format:"csv",fields:["timestamp","name","data"]},out:[["fs.writeFile","events.csv",'1700000000000,page view,"{""title"":""Hello, \\""World\\"""",""count"":3}"\n']]};//# sourceMappingURL=dev.js.map
|
package/dist/dev.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/dev.ts","../src/schemas/index.ts","../src/schemas/settings.ts","../src/schemas/mapping.ts","../src/examples/index.ts","../src/examples/env.ts","../src/examples/step.ts"],"sourcesContent":["export * as schemas from './schemas';\nexport * as examples from './examples';\n","import { zodToSchema } from '@walkeros/core/dev';\nimport { SettingsSchema } from './settings';\nimport { MappingSchema } from './mapping';\n\nexport { SettingsSchema, type Settings } from './settings';\nexport { MappingSchema, type Mapping } from './mapping';\n\n// JSON Schema\nexport const settings = zodToSchema(SettingsSchema);\nexport const mapping = zodToSchema(MappingSchema);\n","import { z } from '@walkeros/core/dev';\n\nexport const SettingsSchema = z.object({\n filename: z\n .union([\n z.string().describe('Static output filename.'),\n z\n .record(z.string(), z.unknown())\n .describe(\n 'Mapping.Value resolved per event (e.g. { key: \"data.tenant\" } or { fn: \"$code:...\" }).',\n ),\n ])\n .describe(\n 'Output filename. Static string or Mapping.Value (e.g. { fn: \"$code:...\" } for daily rotation, { key: \"data.tenant\" } for sharding).',\n ),\n format: z\n .enum(['jsonl', 'tsv', 'csv'])\n .optional()\n .describe('Serialisation format. Defaults to jsonl.'),\n fields: z\n .array(z.string())\n .optional()\n .describe(\n 'Event paths used as columns for tsv/csv formats. Object values are JSON-stringified. Required when format is tsv or csv.',\n ),\n});\n\nexport type Settings = z.infer<typeof SettingsSchema>;\n","import { z } from '@walkeros/core/dev';\n\nexport const MappingSchema = z\n .object({})\n .describe('No per-rule overrides for the file destination.');\n\nexport type Mapping = z.infer<typeof MappingSchema>;\n","export * as env from './env';\nexport * as step from './step';\n","import type { Env, FileWriteStream } from '../types';\n\n/**\n * Captured file state for assertions in tests.\n */\nexport interface CapturedFile {\n filename: string;\n lines: string[];\n ended: boolean;\n}\n\n// Narrow helper type aliases so the mock fs is typed explicitly without `any`.\ntype CreateWriteStreamFn = (\n path: string,\n options: { flags: string },\n) => FileWriteStream;\ntype MkdirFn = (path: string, options: { recursive: boolean }) => Promise<void>;\n\nexport interface SpyState {\n captured: Map<string, CapturedFile>;\n mkdirCalls: string[];\n}\n\nexport interface SpyEnv extends Env {\n _spy: SpyState;\n}\n\nfunction makeSpyEnv(): SpyEnv {\n const state: SpyState = {\n captured: new Map(),\n mkdirCalls: [],\n };\n\n const createWriteStream: CreateWriteStreamFn = (path) => {\n const existing = state.captured.get(path);\n const file: CapturedFile = existing ?? {\n filename: path,\n lines: [],\n ended: false,\n };\n if (!existing) state.captured.set(path, file);\n const stream: FileWriteStream = {\n write(chunk) {\n file.lines.push(chunk);\n return true;\n },\n end() {\n file.ended = true;\n },\n };\n return stream;\n };\n\n const mkdir: MkdirFn = async (path) => {\n state.mkdirCalls.push(path);\n };\n\n return {\n _spy: state,\n fs: {\n createWriteStream,\n mkdir,\n },\n };\n}\n\nexport const init: SpyEnv = makeSpyEnv();\nexport const push: SpyEnv = makeSpyEnv();\n","import type { Flow } from '@walkeros/core';\nimport { getEvent } from '@walkeros/core';\nimport type { Format } from '../types';\n\n/**\n * Raw filename-config shape accepted in step examples and flow.json.\n * Mirrors the runtime Settings.filename but types `fn` as string so\n * `$code:` tags (resolved at bundle time by walkerOS) are expressible\n * here without casting to a function type.\n */\nexport interface FileSettingsJson {\n filename:\n | string\n | {\n key?: string;\n fn?: string | ((value: unknown) => unknown);\n value?: unknown;\n };\n format?: Format;\n fields?: string[];\n}\n\n/**\n * Extended step example that carries destination-level settings for the\n * file destination.\n *\n * `out` is modeled at intent level as a single `['fs.writeFile', path, line]`\n * tuple per event (even though the destination internally uses a persistent\n * createWriteStream + write). This matches the design doc's preference for\n * readable intent-level callable tuples over stream-chain fidelity.\n *\n * Dropped / ignored events produce `out: []`.\n */\nexport type FileStepExample = Flow.StepExample & {\n settings: FileSettingsJson;\n};\n\n/** Default JSONL append. Static filename, all defaults. */\nexport const jsonlDefault: FileStepExample = {\n in: getEvent('page view', { timestamp: 1700000000000 }),\n settings: { filename: 'events.jsonl' },\n out: [\n [\n 'fs.writeFile',\n 'events.jsonl',\n // JSONL framework contract: JSON.stringify(event) + '\\n'.\n // The test recomputes this from example.in for the jsonl default to\n // avoid duplicating the full event shape here.\n 'jsonl:event',\n ],\n ],\n};\n\n/** Baersch-style TSV log: PHP parity case. */\nexport const tsvBaerschLog: FileStepExample = {\n in: getEvent('page view', {\n timestamp: 1700000000000,\n data: { title: 'Docs' },\n user: { session: 'sess-1' },\n source: {\n id: 'https://example.com/docs',\n type: 'server',\n previous_id: 'https://example.com/',\n },\n }),\n settings: {\n filename: 'storage/mblog.txt',\n format: 'tsv',\n fields: [\n 'timestamp',\n 'user.session',\n 'name',\n 'source.id',\n 'data.title',\n 'source.previous_id',\n ],\n },\n out: [\n [\n 'fs.writeFile',\n 'storage/mblog.txt',\n '1700000000000\\tsess-1\\tpage view\\thttps://example.com/docs\\tDocs\\thttps://example.com/\\n',\n ],\n ],\n};\n\n/** Tenant sharding via plain key extraction. */\nexport const jsonlTenantShardKey: FileStepExample = {\n in: getEvent('custom event', {\n timestamp: 1700000000000,\n data: { tenant: 'acme' },\n }),\n settings: {\n filename: { key: 'data.tenant' },\n format: 'jsonl',\n },\n out: [['fs.writeFile', 'acme', 'jsonl:event']],\n};\n\n/**\n * Daily rotation via a mapping fn. In flow.json this is authored as a\n * `$code:` string; walkerOS compiles it to a function at bundle time. For\n * in-process tests we pass the already-compiled function directly so the\n * example exercises the same code path without needing the bundler.\n */\nexport const jsonlDailyRotation: FileStepExample = {\n in: getEvent('order complete', {\n timestamp: Date.UTC(2026, 3, 15, 12, 34, 56),\n data: { id: 'ORD-1' },\n }),\n settings: {\n filename: {\n fn: (value) => {\n const event = value as { timestamp?: number };\n const ts = event.timestamp ?? 0;\n return `events-${new Date(ts).toISOString().slice(0, 10)}.jsonl`;\n },\n },\n format: 'jsonl',\n },\n out: [['fs.writeFile', 'events-2026-04-15.jsonl', 'jsonl:event']],\n};\n\n/** CSV with an object cell. data is JSON-stringified, properly quoted. */\nexport const csvObjectCell: FileStepExample = {\n in: getEvent('page view', {\n timestamp: 1700000000000,\n data: { title: 'Hello, \"World\"', count: 3 },\n }),\n settings: {\n filename: 'events.csv',\n format: 'csv',\n fields: ['timestamp', 'name', 'data'],\n },\n out: [\n [\n 'fs.writeFile',\n 'events.csv',\n '1700000000000,page view,\"{\"\"title\"\":\"\"Hello, \\\\\"\"World\\\\\"\"\"\",\"\"count\"\":3}\"\\n',\n ],\n ],\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAAAA,cAA4B;;;ACA5B,iBAAkB;AAEX,IAAM,iBAAiB,aAAE,OAAO;AAAA,EACrC,UAAU,aACP,MAAM;AAAA,IACL,aAAE,OAAO,EAAE,SAAS,yBAAyB;AAAA,IAC7C,aACG,OAAO,aAAE,OAAO,GAAG,aAAE,QAAQ,CAAC,EAC9B;AAAA,MACC;AAAA,IACF;AAAA,EACJ,CAAC,EACA;AAAA,IACC;AAAA,EACF;AAAA,EACF,QAAQ,aACL,KAAK,CAAC,SAAS,OAAO,KAAK,CAAC,EAC5B,SAAS,EACT,SAAS,0CAA0C;AAAA,EACtD,QAAQ,aACL,MAAM,aAAE,OAAO,CAAC,EAChB,SAAS,EACT;AAAA,IACC;AAAA,EACF;AACJ,CAAC;;;ACzBD,IAAAC,cAAkB;AAEX,IAAM,gBAAgB,cAC1B,OAAO,CAAC,CAAC,EACT,SAAS,iDAAiD;;;AFItD,IAAM,eAAW,yBAAY,cAAc;AAC3C,IAAM,cAAU,yBAAY,aAAa;;;AGThD;AAAA;AAAA;AAAA;AAAA;;;ACAA;AAAA;AAAA;AAAA;AAAA;AA2BA,SAAS,aAAqB;AAC5B,QAAM,QAAkB;AAAA,IACtB,UAAU,oBAAI,IAAI;AAAA,IAClB,YAAY,CAAC;AAAA,EACf;AAEA,QAAM,oBAAyC,CAAC,SAAS;AACvD,UAAM,WAAW,MAAM,SAAS,IAAI,IAAI;AACxC,UAAM,OAAqB,8BAAY;AAAA,MACrC,UAAU;AAAA,MACV,OAAO,CAAC;AAAA,MACR,OAAO;AAAA,IACT;AACA,QAAI,CAAC,SAAU,OAAM,SAAS,IAAI,MAAM,IAAI;AAC5C,UAAM,SAA0B;AAAA,MAC9B,MAAM,OAAO;AACX,aAAK,MAAM,KAAK,KAAK;AACrB,eAAO;AAAA,MACT;AAAA,MACA,MAAM;AACJ,aAAK,QAAQ;AAAA,MACf;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,QAAM,QAAiB,OAAO,SAAS;AACrC,UAAM,WAAW,KAAK,IAAI;AAAA,EAC5B;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN,IAAI;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,IAAM,OAAe,WAAW;AAChC,IAAM,OAAe,WAAW;;;ACnEvC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,kBAAyB;AAqClB,IAAM,eAAgC;AAAA,EAC3C,QAAI,sBAAS,aAAa,EAAE,WAAW,MAAc,CAAC;AAAA,EACtD,UAAU,EAAE,UAAU,eAAe;AAAA,EACrC,KAAK;AAAA,IACH;AAAA,MACE;AAAA,MACA;AAAA;AAAA;AAAA;AAAA,MAIA;AAAA,IACF;AAAA,EACF;AACF;AAGO,IAAM,gBAAiC;AAAA,EAC5C,QAAI,sBAAS,aAAa;AAAA,IACxB,WAAW;AAAA,IACX,MAAM,EAAE,OAAO,OAAO;AAAA,IACtB,MAAM,EAAE,SAAS,SAAS;AAAA,IAC1B,QAAQ;AAAA,MACN,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,aAAa;AAAA,IACf;AAAA,EACF,CAAC;AAAA,EACD,UAAU;AAAA,IACR,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EACA,KAAK;AAAA,IACH;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAGO,IAAM,sBAAuC;AAAA,EAClD,QAAI,sBAAS,gBAAgB;AAAA,IAC3B,WAAW;AAAA,IACX,MAAM,EAAE,QAAQ,OAAO;AAAA,EACzB,CAAC;AAAA,EACD,UAAU;AAAA,IACR,UAAU,EAAE,KAAK,cAAc;AAAA,IAC/B,QAAQ;AAAA,EACV;AAAA,EACA,KAAK,CAAC,CAAC,gBAAgB,QAAQ,aAAa,CAAC;AAC/C;AAQO,IAAM,qBAAsC;AAAA,EACjD,QAAI,sBAAS,kBAAkB;AAAA,IAC7B,WAAW,KAAK,IAAI,MAAM,GAAG,IAAI,IAAI,IAAI,EAAE;AAAA,IAC3C,MAAM,EAAE,IAAI,QAAQ;AAAA,EACtB,CAAC;AAAA,EACD,UAAU;AAAA,IACR,UAAU;AAAA,MACR,IAAI,CAAC,UAAU;AAhHrB;AAiHQ,cAAM,QAAQ;AACd,cAAM,MAAK,WAAM,cAAN,YAAmB;AAC9B,eAAO,UAAU,IAAI,KAAK,EAAE,EAAE,YAAY,EAAE,MAAM,GAAG,EAAE,CAAC;AAAA,MAC1D;AAAA,IACF;AAAA,IACA,QAAQ;AAAA,EACV;AAAA,EACA,KAAK,CAAC,CAAC,gBAAgB,2BAA2B,aAAa,CAAC;AAClE;AAGO,IAAM,gBAAiC;AAAA,EAC5C,QAAI,sBAAS,aAAa;AAAA,IACxB,WAAW;AAAA,IACX,MAAM,EAAE,OAAO,kBAAkB,OAAO,EAAE;AAAA,EAC5C,CAAC;AAAA,EACD,UAAU;AAAA,IACR,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,QAAQ,CAAC,aAAa,QAAQ,MAAM;AAAA,EACtC;AAAA,EACA,KAAK;AAAA,IACH;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;","names":["import_dev","import_dev"]}
|
package/dist/dev.mjs
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
var e=Object.defineProperty,t=(t,s)=>{for(var a in s)e(t,a,{get:s[a],enumerable:!0})},s={};t(s,{MappingSchema:()=>r,SettingsSchema:()=>i,mapping:()=>m,settings:()=>l});import{zodToSchema as a}from"@walkeros/core/dev";import{z as n}from"@walkeros/core/dev";var i=n.object({filename:n.union([n.string().describe("Static output filename."),n.record(n.string(),n.unknown()).describe('Mapping.Value resolved per event (e.g. { key: "data.tenant" } or { fn: "$code:..." }).')]).describe('Output filename. Static string or Mapping.Value (e.g. { fn: "$code:..." } for daily rotation, { key: "data.tenant" } for sharding).'),format:n.enum(["jsonl","tsv","csv"]).optional().describe("Serialisation format. Defaults to jsonl."),fields:n.array(n.string()).optional().describe("Event paths used as columns for tsv/csv formats. Object values are JSON-stringified. Required when format is tsv or csv.")});import{z as o}from"@walkeros/core/dev";var r=o.object({}).describe("No per-rule overrides for the file destination."),l=a(i),m=a(r),c={};t(c,{env:()=>p,step:()=>u});var p={};function d(){const e={captured:new Map,mkdirCalls:[]};return{_spy:e,fs:{createWriteStream:t=>{const s=e.captured.get(t),a=null!=s?s:{filename:t,lines:[],ended:!1};s||e.captured.set(t,a);return{write:e=>(a.lines.push(e),!0),end(){a.ended=!0}}},mkdir:async t=>{e.mkdirCalls.push(t)}}}}t(p,{init:()=>v,push:()=>f});var v=d(),f=d(),u={};t(u,{csvObjectCell:()=>k,jsonlDailyRotation:()=>b,jsonlDefault:()=>j,jsonlTenantShardKey:()=>h,tsvBaerschLog:()=>w});import{getEvent as g}from"@walkeros/core";var j={in:g("page view",{timestamp:17e11}),settings:{filename:"events.jsonl"},out:[["fs.writeFile","events.jsonl","jsonl:event"]]},w={in:g("page view",{timestamp:17e11,data:{title:"Docs"},user:{session:"sess-1"},source:{id:"https://example.com/docs",type:"server",previous_id:"https://example.com/"}}),settings:{filename:"storage/mblog.txt",format:"tsv",fields:["timestamp","user.session","name","source.id","data.title","source.previous_id"]},out:[["fs.writeFile","storage/mblog.txt","1700000000000\tsess-1\tpage view\thttps://example.com/docs\tDocs\thttps://example.com/\n"]]},h={in:g("custom event",{timestamp:17e11,data:{tenant:"acme"}}),settings:{filename:{key:"data.tenant"},format:"jsonl"},out:[["fs.writeFile","acme","jsonl:event"]]},b={in:g("order complete",{timestamp:Date.UTC(2026,3,15,12,34,56),data:{id:"ORD-1"}}),settings:{filename:{fn:e=>{var t;const s=null!=(t=e.timestamp)?t:0;return`events-${new Date(s).toISOString().slice(0,10)}.jsonl`}},format:"jsonl"},out:[["fs.writeFile","events-2026-04-15.jsonl","jsonl:event"]]},k={in:g("page view",{timestamp:17e11,data:{title:'Hello, "World"',count:3}}),settings:{filename:"events.csv",format:"csv",fields:["timestamp","name","data"]},out:[["fs.writeFile","events.csv",'1700000000000,page view,"{""title"":""Hello, \\""World\\"""",""count"":3}"\n']]};export{c as examples,s as schemas};//# sourceMappingURL=dev.mjs.map
|
package/dist/dev.mjs.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/schemas/index.ts","../src/schemas/settings.ts","../src/schemas/mapping.ts","../src/examples/index.ts","../src/examples/env.ts","../src/examples/step.ts"],"sourcesContent":["import { zodToSchema } from '@walkeros/core/dev';\nimport { SettingsSchema } from './settings';\nimport { MappingSchema } from './mapping';\n\nexport { SettingsSchema, type Settings } from './settings';\nexport { MappingSchema, type Mapping } from './mapping';\n\n// JSON Schema\nexport const settings = zodToSchema(SettingsSchema);\nexport const mapping = zodToSchema(MappingSchema);\n","import { z } from '@walkeros/core/dev';\n\nexport const SettingsSchema = z.object({\n filename: z\n .union([\n z.string().describe('Static output filename.'),\n z\n .record(z.string(), z.unknown())\n .describe(\n 'Mapping.Value resolved per event (e.g. { key: \"data.tenant\" } or { fn: \"$code:...\" }).',\n ),\n ])\n .describe(\n 'Output filename. Static string or Mapping.Value (e.g. { fn: \"$code:...\" } for daily rotation, { key: \"data.tenant\" } for sharding).',\n ),\n format: z\n .enum(['jsonl', 'tsv', 'csv'])\n .optional()\n .describe('Serialisation format. Defaults to jsonl.'),\n fields: z\n .array(z.string())\n .optional()\n .describe(\n 'Event paths used as columns for tsv/csv formats. Object values are JSON-stringified. Required when format is tsv or csv.',\n ),\n});\n\nexport type Settings = z.infer<typeof SettingsSchema>;\n","import { z } from '@walkeros/core/dev';\n\nexport const MappingSchema = z\n .object({})\n .describe('No per-rule overrides for the file destination.');\n\nexport type Mapping = z.infer<typeof MappingSchema>;\n","export * as env from './env';\nexport * as step from './step';\n","import type { Env, FileWriteStream } from '../types';\n\n/**\n * Captured file state for assertions in tests.\n */\nexport interface CapturedFile {\n filename: string;\n lines: string[];\n ended: boolean;\n}\n\n// Narrow helper type aliases so the mock fs is typed explicitly without `any`.\ntype CreateWriteStreamFn = (\n path: string,\n options: { flags: string },\n) => FileWriteStream;\ntype MkdirFn = (path: string, options: { recursive: boolean }) => Promise<void>;\n\nexport interface SpyState {\n captured: Map<string, CapturedFile>;\n mkdirCalls: string[];\n}\n\nexport interface SpyEnv extends Env {\n _spy: SpyState;\n}\n\nfunction makeSpyEnv(): SpyEnv {\n const state: SpyState = {\n captured: new Map(),\n mkdirCalls: [],\n };\n\n const createWriteStream: CreateWriteStreamFn = (path) => {\n const existing = state.captured.get(path);\n const file: CapturedFile = existing ?? {\n filename: path,\n lines: [],\n ended: false,\n };\n if (!existing) state.captured.set(path, file);\n const stream: FileWriteStream = {\n write(chunk) {\n file.lines.push(chunk);\n return true;\n },\n end() {\n file.ended = true;\n },\n };\n return stream;\n };\n\n const mkdir: MkdirFn = async (path) => {\n state.mkdirCalls.push(path);\n };\n\n return {\n _spy: state,\n fs: {\n createWriteStream,\n mkdir,\n },\n };\n}\n\nexport const init: SpyEnv = makeSpyEnv();\nexport const push: SpyEnv = makeSpyEnv();\n","import type { Flow } from '@walkeros/core';\nimport { getEvent } from '@walkeros/core';\nimport type { Format } from '../types';\n\n/**\n * Raw filename-config shape accepted in step examples and flow.json.\n * Mirrors the runtime Settings.filename but types `fn` as string so\n * `$code:` tags (resolved at bundle time by walkerOS) are expressible\n * here without casting to a function type.\n */\nexport interface FileSettingsJson {\n filename:\n | string\n | {\n key?: string;\n fn?: string | ((value: unknown) => unknown);\n value?: unknown;\n };\n format?: Format;\n fields?: string[];\n}\n\n/**\n * Extended step example that carries destination-level settings for the\n * file destination.\n *\n * `out` is modeled at intent level as a single `['fs.writeFile', path, line]`\n * tuple per event (even though the destination internally uses a persistent\n * createWriteStream + write). This matches the design doc's preference for\n * readable intent-level callable tuples over stream-chain fidelity.\n *\n * Dropped / ignored events produce `out: []`.\n */\nexport type FileStepExample = Flow.StepExample & {\n settings: FileSettingsJson;\n};\n\n/** Default JSONL append. Static filename, all defaults. */\nexport const jsonlDefault: FileStepExample = {\n in: getEvent('page view', { timestamp: 1700000000000 }),\n settings: { filename: 'events.jsonl' },\n out: [\n [\n 'fs.writeFile',\n 'events.jsonl',\n // JSONL framework contract: JSON.stringify(event) + '\\n'.\n // The test recomputes this from example.in for the jsonl default to\n // avoid duplicating the full event shape here.\n 'jsonl:event',\n ],\n ],\n};\n\n/** Baersch-style TSV log: PHP parity case. */\nexport const tsvBaerschLog: FileStepExample = {\n in: getEvent('page view', {\n timestamp: 1700000000000,\n data: { title: 'Docs' },\n user: { session: 'sess-1' },\n source: {\n id: 'https://example.com/docs',\n type: 'server',\n previous_id: 'https://example.com/',\n },\n }),\n settings: {\n filename: 'storage/mblog.txt',\n format: 'tsv',\n fields: [\n 'timestamp',\n 'user.session',\n 'name',\n 'source.id',\n 'data.title',\n 'source.previous_id',\n ],\n },\n out: [\n [\n 'fs.writeFile',\n 'storage/mblog.txt',\n '1700000000000\\tsess-1\\tpage view\\thttps://example.com/docs\\tDocs\\thttps://example.com/\\n',\n ],\n ],\n};\n\n/** Tenant sharding via plain key extraction. */\nexport const jsonlTenantShardKey: FileStepExample = {\n in: getEvent('custom event', {\n timestamp: 1700000000000,\n data: { tenant: 'acme' },\n }),\n settings: {\n filename: { key: 'data.tenant' },\n format: 'jsonl',\n },\n out: [['fs.writeFile', 'acme', 'jsonl:event']],\n};\n\n/**\n * Daily rotation via a mapping fn. In flow.json this is authored as a\n * `$code:` string; walkerOS compiles it to a function at bundle time. For\n * in-process tests we pass the already-compiled function directly so the\n * example exercises the same code path without needing the bundler.\n */\nexport const jsonlDailyRotation: FileStepExample = {\n in: getEvent('order complete', {\n timestamp: Date.UTC(2026, 3, 15, 12, 34, 56),\n data: { id: 'ORD-1' },\n }),\n settings: {\n filename: {\n fn: (value) => {\n const event = value as { timestamp?: number };\n const ts = event.timestamp ?? 0;\n return `events-${new Date(ts).toISOString().slice(0, 10)}.jsonl`;\n },\n },\n format: 'jsonl',\n },\n out: [['fs.writeFile', 'events-2026-04-15.jsonl', 'jsonl:event']],\n};\n\n/** CSV with an object cell. data is JSON-stringified, properly quoted. */\nexport const csvObjectCell: FileStepExample = {\n in: getEvent('page view', {\n timestamp: 1700000000000,\n data: { title: 'Hello, \"World\"', count: 3 },\n }),\n settings: {\n filename: 'events.csv',\n format: 'csv',\n fields: ['timestamp', 'name', 'data'],\n },\n out: [\n [\n 'fs.writeFile',\n 'events.csv',\n '1700000000000,page view,\"{\"\"title\"\":\"\"Hello, \\\\\"\"World\\\\\"\"\"\",\"\"count\"\":3}\"\\n',\n ],\n ],\n};\n"],"mappings":";;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,SAAS,mBAAmB;;;ACA5B,SAAS,SAAS;AAEX,IAAM,iBAAiB,EAAE,OAAO;AAAA,EACrC,UAAU,EACP,MAAM;AAAA,IACL,EAAE,OAAO,EAAE,SAAS,yBAAyB;AAAA,IAC7C,EACG,OAAO,EAAE,OAAO,GAAG,EAAE,QAAQ,CAAC,EAC9B;AAAA,MACC;AAAA,IACF;AAAA,EACJ,CAAC,EACA;AAAA,IACC;AAAA,EACF;AAAA,EACF,QAAQ,EACL,KAAK,CAAC,SAAS,OAAO,KAAK,CAAC,EAC5B,SAAS,EACT,SAAS,0CAA0C;AAAA,EACtD,QAAQ,EACL,MAAM,EAAE,OAAO,CAAC,EAChB,SAAS,EACT;AAAA,IACC;AAAA,EACF;AACJ,CAAC;;;ACzBD,SAAS,KAAAA,UAAS;AAEX,IAAM,gBAAgBA,GAC1B,OAAO,CAAC,CAAC,EACT,SAAS,iDAAiD;;;AFItD,IAAM,WAAW,YAAY,cAAc;AAC3C,IAAM,UAAU,YAAY,aAAa;;;AGThD;AAAA;AAAA;AAAA;AAAA;;;ACAA;AAAA;AAAA;AAAA;AAAA;AA2BA,SAAS,aAAqB;AAC5B,QAAM,QAAkB;AAAA,IACtB,UAAU,oBAAI,IAAI;AAAA,IAClB,YAAY,CAAC;AAAA,EACf;AAEA,QAAM,oBAAyC,CAAC,SAAS;AACvD,UAAM,WAAW,MAAM,SAAS,IAAI,IAAI;AACxC,UAAM,OAAqB,8BAAY;AAAA,MACrC,UAAU;AAAA,MACV,OAAO,CAAC;AAAA,MACR,OAAO;AAAA,IACT;AACA,QAAI,CAAC,SAAU,OAAM,SAAS,IAAI,MAAM,IAAI;AAC5C,UAAM,SAA0B;AAAA,MAC9B,MAAM,OAAO;AACX,aAAK,MAAM,KAAK,KAAK;AACrB,eAAO;AAAA,MACT;AAAA,MACA,MAAM;AACJ,aAAK,QAAQ;AAAA,MACf;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,QAAM,QAAiB,OAAO,SAAS;AACrC,UAAM,WAAW,KAAK,IAAI;AAAA,EAC5B;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN,IAAI;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,IAAM,OAAe,WAAW;AAChC,IAAM,OAAe,WAAW;;;ACnEvC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,SAAS,gBAAgB;AAqClB,IAAM,eAAgC;AAAA,EAC3C,IAAI,SAAS,aAAa,EAAE,WAAW,MAAc,CAAC;AAAA,EACtD,UAAU,EAAE,UAAU,eAAe;AAAA,EACrC,KAAK;AAAA,IACH;AAAA,MACE;AAAA,MACA;AAAA;AAAA;AAAA;AAAA,MAIA;AAAA,IACF;AAAA,EACF;AACF;AAGO,IAAM,gBAAiC;AAAA,EAC5C,IAAI,SAAS,aAAa;AAAA,IACxB,WAAW;AAAA,IACX,MAAM,EAAE,OAAO,OAAO;AAAA,IACtB,MAAM,EAAE,SAAS,SAAS;AAAA,IAC1B,QAAQ;AAAA,MACN,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,aAAa;AAAA,IACf;AAAA,EACF,CAAC;AAAA,EACD,UAAU;AAAA,IACR,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EACA,KAAK;AAAA,IACH;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAGO,IAAM,sBAAuC;AAAA,EAClD,IAAI,SAAS,gBAAgB;AAAA,IAC3B,WAAW;AAAA,IACX,MAAM,EAAE,QAAQ,OAAO;AAAA,EACzB,CAAC;AAAA,EACD,UAAU;AAAA,IACR,UAAU,EAAE,KAAK,cAAc;AAAA,IAC/B,QAAQ;AAAA,EACV;AAAA,EACA,KAAK,CAAC,CAAC,gBAAgB,QAAQ,aAAa,CAAC;AAC/C;AAQO,IAAM,qBAAsC;AAAA,EACjD,IAAI,SAAS,kBAAkB;AAAA,IAC7B,WAAW,KAAK,IAAI,MAAM,GAAG,IAAI,IAAI,IAAI,EAAE;AAAA,IAC3C,MAAM,EAAE,IAAI,QAAQ;AAAA,EACtB,CAAC;AAAA,EACD,UAAU;AAAA,IACR,UAAU;AAAA,MACR,IAAI,CAAC,UAAU;AAhHrB;AAiHQ,cAAM,QAAQ;AACd,cAAM,MAAK,WAAM,cAAN,YAAmB;AAC9B,eAAO,UAAU,IAAI,KAAK,EAAE,EAAE,YAAY,EAAE,MAAM,GAAG,EAAE,CAAC;AAAA,MAC1D;AAAA,IACF;AAAA,IACA,QAAQ;AAAA,EACV;AAAA,EACA,KAAK,CAAC,CAAC,gBAAgB,2BAA2B,aAAa,CAAC;AAClE;AAGO,IAAM,gBAAiC;AAAA,EAC5C,IAAI,SAAS,aAAa;AAAA,IACxB,WAAW;AAAA,IACX,MAAM,EAAE,OAAO,kBAAkB,OAAO,EAAE;AAAA,EAC5C,CAAC;AAAA,EACD,UAAU;AAAA,IACR,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,QAAQ,CAAC,aAAa,QAAQ,MAAM;AAAA,EACtC;AAAA,EACA,KAAK;AAAA,IACH;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;","names":["z"]}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { DestinationServer } from '@walkeros/server-core';
|
|
2
|
+
import { WriteStream } from 'node:fs';
|
|
3
|
+
import { Flow } from '@walkeros/core';
|
|
4
|
+
|
|
5
|
+
type Format = 'jsonl' | 'tsv' | 'csv';
|
|
6
|
+
/**
|
|
7
|
+
* Minimal write-stream interface used by the destination. Matches the
|
|
8
|
+
* subset of node:fs WriteStream needed for append-only writes. Tests
|
|
9
|
+
* inject a fake implementation via env.fs.
|
|
10
|
+
*/
|
|
11
|
+
interface FileWriteStream {
|
|
12
|
+
write: (chunk: string) => boolean;
|
|
13
|
+
end: () => void;
|
|
14
|
+
}
|
|
15
|
+
interface Env extends DestinationServer.Env {
|
|
16
|
+
/**
|
|
17
|
+
* Override the file system primitives. Tests inject a fake here so
|
|
18
|
+
* disk writes are captured in memory.
|
|
19
|
+
*/
|
|
20
|
+
fs?: {
|
|
21
|
+
createWriteStream: (path: string, options: {
|
|
22
|
+
flags: string;
|
|
23
|
+
}) => FileWriteStream | WriteStream;
|
|
24
|
+
mkdir: (path: string, options: {
|
|
25
|
+
recursive: boolean;
|
|
26
|
+
}) => Promise<void>;
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Captured file state for assertions in tests.
|
|
32
|
+
*/
|
|
33
|
+
interface CapturedFile {
|
|
34
|
+
filename: string;
|
|
35
|
+
lines: string[];
|
|
36
|
+
ended: boolean;
|
|
37
|
+
}
|
|
38
|
+
interface SpyState {
|
|
39
|
+
captured: Map<string, CapturedFile>;
|
|
40
|
+
mkdirCalls: string[];
|
|
41
|
+
}
|
|
42
|
+
interface SpyEnv extends Env {
|
|
43
|
+
_spy: SpyState;
|
|
44
|
+
}
|
|
45
|
+
declare const init: SpyEnv;
|
|
46
|
+
declare const push: SpyEnv;
|
|
47
|
+
|
|
48
|
+
type env_CapturedFile = CapturedFile;
|
|
49
|
+
type env_SpyEnv = SpyEnv;
|
|
50
|
+
type env_SpyState = SpyState;
|
|
51
|
+
declare const env_init: typeof init;
|
|
52
|
+
declare const env_push: typeof push;
|
|
53
|
+
declare namespace env {
|
|
54
|
+
export { type env_CapturedFile as CapturedFile, type env_SpyEnv as SpyEnv, type env_SpyState as SpyState, env_init as init, env_push as push };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Raw filename-config shape accepted in step examples and flow.json.
|
|
59
|
+
* Mirrors the runtime Settings.filename but types `fn` as string so
|
|
60
|
+
* `$code:` tags (resolved at bundle time by walkerOS) are expressible
|
|
61
|
+
* here without casting to a function type.
|
|
62
|
+
*/
|
|
63
|
+
interface FileSettingsJson {
|
|
64
|
+
filename: string | {
|
|
65
|
+
key?: string;
|
|
66
|
+
fn?: string | ((value: unknown) => unknown);
|
|
67
|
+
value?: unknown;
|
|
68
|
+
};
|
|
69
|
+
format?: Format;
|
|
70
|
+
fields?: string[];
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Extended step example that carries destination-level settings for the
|
|
74
|
+
* file destination.
|
|
75
|
+
*
|
|
76
|
+
* `out` is modeled at intent level as a single `['fs.writeFile', path, line]`
|
|
77
|
+
* tuple per event (even though the destination internally uses a persistent
|
|
78
|
+
* createWriteStream + write). This matches the design doc's preference for
|
|
79
|
+
* readable intent-level callable tuples over stream-chain fidelity.
|
|
80
|
+
*
|
|
81
|
+
* Dropped / ignored events produce `out: []`.
|
|
82
|
+
*/
|
|
83
|
+
type FileStepExample = Flow.StepExample & {
|
|
84
|
+
settings: FileSettingsJson;
|
|
85
|
+
};
|
|
86
|
+
/** Default JSONL append. Static filename, all defaults. */
|
|
87
|
+
declare const jsonlDefault: FileStepExample;
|
|
88
|
+
/** Baersch-style TSV log: PHP parity case. */
|
|
89
|
+
declare const tsvBaerschLog: FileStepExample;
|
|
90
|
+
/** Tenant sharding via plain key extraction. */
|
|
91
|
+
declare const jsonlTenantShardKey: FileStepExample;
|
|
92
|
+
/**
|
|
93
|
+
* Daily rotation via a mapping fn. In flow.json this is authored as a
|
|
94
|
+
* `$code:` string; walkerOS compiles it to a function at bundle time. For
|
|
95
|
+
* in-process tests we pass the already-compiled function directly so the
|
|
96
|
+
* example exercises the same code path without needing the bundler.
|
|
97
|
+
*/
|
|
98
|
+
declare const jsonlDailyRotation: FileStepExample;
|
|
99
|
+
/** CSV with an object cell. data is JSON-stringified, properly quoted. */
|
|
100
|
+
declare const csvObjectCell: FileStepExample;
|
|
101
|
+
|
|
102
|
+
type step_FileSettingsJson = FileSettingsJson;
|
|
103
|
+
type step_FileStepExample = FileStepExample;
|
|
104
|
+
declare const step_csvObjectCell: typeof csvObjectCell;
|
|
105
|
+
declare const step_jsonlDailyRotation: typeof jsonlDailyRotation;
|
|
106
|
+
declare const step_jsonlDefault: typeof jsonlDefault;
|
|
107
|
+
declare const step_jsonlTenantShardKey: typeof jsonlTenantShardKey;
|
|
108
|
+
declare const step_tsvBaerschLog: typeof tsvBaerschLog;
|
|
109
|
+
declare namespace step {
|
|
110
|
+
export { type step_FileSettingsJson as FileSettingsJson, type step_FileStepExample as FileStepExample, step_csvObjectCell as csvObjectCell, step_jsonlDailyRotation as jsonlDailyRotation, step_jsonlDefault as jsonlDefault, step_jsonlTenantShardKey as jsonlTenantShardKey, step_tsvBaerschLog as tsvBaerschLog };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export { env, step };
|