@mepuka/skygent 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +269 -31
- package/index.ts +18 -3
- package/package.json +1 -1
- package/src/cli/app.ts +4 -2
- package/src/cli/compact-output.ts +52 -0
- package/src/cli/config.ts +46 -4
- package/src/cli/doc/table-renderers.ts +29 -0
- package/src/cli/doc/thread.ts +2 -4
- package/src/cli/exit-codes.ts +2 -0
- package/src/cli/feed.ts +78 -61
- package/src/cli/filter-dsl.ts +146 -11
- package/src/cli/filter-errors.ts +13 -11
- package/src/cli/filter-help.ts +7 -0
- package/src/cli/filter-input.ts +3 -2
- package/src/cli/filter.ts +83 -5
- package/src/cli/graph.ts +297 -169
- package/src/cli/input.ts +45 -0
- package/src/cli/interval.ts +4 -33
- package/src/cli/jetstream.ts +2 -0
- package/src/cli/layers.ts +10 -0
- package/src/cli/logging.ts +8 -0
- package/src/cli/option-schemas.ts +22 -0
- package/src/cli/output-format.ts +11 -0
- package/src/cli/output-render.ts +14 -0
- package/src/cli/pagination.ts +17 -0
- package/src/cli/parse-errors.ts +30 -0
- package/src/cli/parse.ts +1 -47
- package/src/cli/pipe-input.ts +18 -0
- package/src/cli/pipe.ts +154 -0
- package/src/cli/post.ts +88 -66
- package/src/cli/query-fields.ts +13 -3
- package/src/cli/query.ts +354 -100
- package/src/cli/search.ts +93 -136
- package/src/cli/shared-options.ts +11 -63
- package/src/cli/shared.ts +1 -20
- package/src/cli/store-errors.ts +28 -21
- package/src/cli/store-tree.ts +6 -4
- package/src/cli/store.ts +41 -2
- package/src/cli/stream-merge.ts +105 -0
- package/src/cli/sync-factory.ts +24 -7
- package/src/cli/sync.ts +46 -67
- package/src/cli/thread-options.ts +25 -0
- package/src/cli/time.ts +171 -0
- package/src/cli/view-thread.ts +29 -32
- package/src/cli/watch.ts +55 -26
- package/src/domain/errors.ts +6 -1
- package/src/domain/format.ts +21 -0
- package/src/domain/order.ts +24 -0
- package/src/domain/primitives.ts +20 -3
- package/src/graph/relationships.ts +129 -0
- package/src/services/bsky-client.ts +11 -5
- package/src/services/jetstream-sync.ts +4 -4
- package/src/services/lineage-store.ts +15 -1
- package/src/services/shared.ts +48 -1
- package/src/services/store-cleaner.ts +5 -2
- package/src/services/store-commit.ts +60 -0
- package/src/services/store-manager.ts +69 -2
- package/src/services/store-renamer.ts +288 -0
- package/src/services/store-stats.ts +7 -5
- package/src/services/sync-engine.ts +149 -89
- package/src/services/sync-reporter.ts +3 -1
- package/src/services/sync-settings.ts +24 -0
package/src/cli/query.ts
CHANGED
|
@@ -1,42 +1,69 @@
|
|
|
1
1
|
import { Args, Command, Options } from "@effect/cli";
|
|
2
|
-
import { Chunk, Clock, Effect, Option, Ref, Stream } from "effect";
|
|
2
|
+
import { Chunk, Clock, Effect, Option, Order, Ref, Schema, Stream } from "effect";
|
|
3
3
|
import * as Doc from "@effect/printer/Doc";
|
|
4
4
|
import { all } from "../domain/filter.js";
|
|
5
5
|
import type { FilterExpr } from "../domain/filter.js";
|
|
6
6
|
import { StoreQuery } from "../domain/events.js";
|
|
7
|
-
import { StoreName } from "../domain/primitives.js";
|
|
7
|
+
import { StoreName, Timestamp } from "../domain/primitives.js";
|
|
8
8
|
import type { Post } from "../domain/post.js";
|
|
9
|
+
import type { StoreRef } from "../domain/store.js";
|
|
9
10
|
import { FilterRuntime } from "../services/filter-runtime.js";
|
|
10
11
|
import { AppConfigService } from "../services/app-config.js";
|
|
11
12
|
import { StoreIndex } from "../services/store-index.js";
|
|
12
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
renderPostsMarkdown,
|
|
15
|
+
renderPostsTable,
|
|
16
|
+
renderStorePostsMarkdown,
|
|
17
|
+
renderStorePostsTable
|
|
18
|
+
} from "../domain/format.js";
|
|
13
19
|
import { renderPostCompact, renderPostCard } from "./doc/post.js";
|
|
14
20
|
import { renderThread } from "./doc/thread.js";
|
|
15
21
|
import { renderPlain, renderAnsi } from "./doc/render.js";
|
|
16
22
|
import { parseOptionalFilterExpr } from "./filter-input.js";
|
|
17
23
|
import { CliOutput, writeJson, writeJsonStream, writeText } from "./output.js";
|
|
18
24
|
import { parseRange } from "./range.js";
|
|
19
|
-
import {
|
|
25
|
+
import { parseTimeInput } from "./time.js";
|
|
20
26
|
import { CliPreferences } from "./preferences.js";
|
|
21
27
|
import { projectFields, resolveFieldSelectors } from "./query-fields.js";
|
|
22
28
|
import { CliInputError } from "./errors.js";
|
|
23
29
|
import { withExamples } from "./help.js";
|
|
24
30
|
import { filterOption, filterJsonOption } from "./shared-options.js";
|
|
25
31
|
import { filterByFlags } from "../typeclass/chunk.js";
|
|
32
|
+
import { StoreManager } from "../services/store-manager.js";
|
|
33
|
+
import { StoreNotFound } from "../domain/errors.js";
|
|
34
|
+
import { StorePostOrder } from "../domain/order.js";
|
|
35
|
+
import { formatSchemaError } from "./shared.js";
|
|
36
|
+
import { mergeOrderedStreams } from "./stream-merge.js";
|
|
37
|
+
import { queryOutputFormats, resolveOutputFormat } from "./output-format.js";
|
|
38
|
+
import { PositiveInt } from "./option-schemas.js";
|
|
26
39
|
|
|
27
|
-
const
|
|
28
|
-
Args.
|
|
29
|
-
Args.withDescription("Store name to query")
|
|
40
|
+
const storeNamesArg = Args.text({ name: "store" }).pipe(
|
|
41
|
+
Args.repeated,
|
|
42
|
+
Args.withDescription("Store name(s) to query (repeatable or comma-separated)")
|
|
30
43
|
);
|
|
31
44
|
const rangeOption = Options.text("range").pipe(
|
|
32
45
|
Options.withDescription("ISO range as <start>..<end>"),
|
|
33
46
|
Options.optional
|
|
34
47
|
);
|
|
48
|
+
const sinceOption = Options.text("since").pipe(
|
|
49
|
+
Options.withDescription(
|
|
50
|
+
"Start time (ISO timestamp, date, relative duration like 24h, or now/today/yesterday)"
|
|
51
|
+
),
|
|
52
|
+
Options.optional
|
|
53
|
+
);
|
|
54
|
+
const untilOption = Options.text("until").pipe(
|
|
55
|
+
Options.withDescription(
|
|
56
|
+
"End time (ISO timestamp, date, relative duration like 24h, or now/today/yesterday)"
|
|
57
|
+
),
|
|
58
|
+
Options.optional
|
|
59
|
+
);
|
|
35
60
|
const limitOption = Options.integer("limit").pipe(
|
|
61
|
+
Options.withSchema(PositiveInt),
|
|
36
62
|
Options.withDescription("Maximum number of posts to return"),
|
|
37
63
|
Options.optional
|
|
38
64
|
);
|
|
39
65
|
const scanLimitOption = Options.integer("scan-limit").pipe(
|
|
66
|
+
Options.withSchema(PositiveInt),
|
|
40
67
|
Options.withDescription("Maximum rows to scan before filtering (advanced)"),
|
|
41
68
|
Options.optional
|
|
42
69
|
);
|
|
@@ -47,22 +74,18 @@ const sortOption = Options.choice("sort", ["asc", "desc"]).pipe(
|
|
|
47
74
|
const newestFirstOption = Options.boolean("newest-first").pipe(
|
|
48
75
|
Options.withDescription("Sort newest posts first (alias for --sort desc)")
|
|
49
76
|
);
|
|
50
|
-
const formatOption = Options.choice("format",
|
|
51
|
-
"json",
|
|
52
|
-
"ndjson",
|
|
53
|
-
"markdown",
|
|
54
|
-
"table",
|
|
55
|
-
"compact",
|
|
56
|
-
"card",
|
|
57
|
-
"thread"
|
|
58
|
-
]).pipe(
|
|
77
|
+
const formatOption = Options.choice("format", queryOutputFormats).pipe(
|
|
59
78
|
Options.optional,
|
|
60
79
|
Options.withDescription("Output format (default: config output format)")
|
|
61
80
|
);
|
|
81
|
+
const includeStoreOption = Options.boolean("include-store").pipe(
|
|
82
|
+
Options.withDescription("Include store name in output")
|
|
83
|
+
);
|
|
62
84
|
const ansiOption = Options.boolean("ansi").pipe(
|
|
63
85
|
Options.withDescription("Enable ANSI colors in output")
|
|
64
86
|
);
|
|
65
87
|
const widthOption = Options.integer("width").pipe(
|
|
88
|
+
Options.withSchema(PositiveInt),
|
|
66
89
|
Options.withDescription("Line width for terminal output"),
|
|
67
90
|
Options.optional
|
|
68
91
|
);
|
|
@@ -75,9 +98,17 @@ const fieldsOption = Options.text("fields").pipe(
|
|
|
75
98
|
const progressOption = Options.boolean("progress").pipe(
|
|
76
99
|
Options.withDescription("Show progress for filtered queries")
|
|
77
100
|
);
|
|
101
|
+
const countOption = Options.boolean("count").pipe(
|
|
102
|
+
Options.withDescription("Only output the count of matching posts")
|
|
103
|
+
);
|
|
78
104
|
|
|
79
105
|
const DEFAULT_FILTER_SCAN_LIMIT = 5000;
|
|
80
106
|
|
|
107
|
+
type StorePost = {
|
|
108
|
+
readonly store: StoreRef;
|
|
109
|
+
readonly post: Post;
|
|
110
|
+
};
|
|
111
|
+
|
|
81
112
|
const isAscii = (value: string) => /^[\x00-\x7F]*$/.test(value);
|
|
82
113
|
|
|
83
114
|
const hasUnicodeInsensitiveContains = (expr: FilterExpr): boolean => {
|
|
@@ -101,18 +132,127 @@ const hasUnicodeInsensitiveContains = (expr: FilterExpr): boolean => {
|
|
|
101
132
|
}
|
|
102
133
|
};
|
|
103
134
|
|
|
104
|
-
const
|
|
105
|
-
Option.
|
|
106
|
-
|
|
107
|
-
|
|
135
|
+
const parseRangeOptions = (
|
|
136
|
+
range: Option.Option<string>,
|
|
137
|
+
since: Option.Option<string>,
|
|
138
|
+
until: Option.Option<string>
|
|
139
|
+
) =>
|
|
140
|
+
Effect.gen(function* () {
|
|
141
|
+
const toTimestamp = (date: Date, label: string) =>
|
|
142
|
+
Schema.decodeUnknown(Timestamp)(date).pipe(
|
|
143
|
+
Effect.mapError((cause) =>
|
|
144
|
+
CliInputError.make({
|
|
145
|
+
message: `Computed ${label} timestamp is invalid.`,
|
|
146
|
+
cause
|
|
147
|
+
})
|
|
148
|
+
)
|
|
149
|
+
);
|
|
150
|
+
const hasRange = Option.isSome(range);
|
|
151
|
+
const hasSince = Option.isSome(since);
|
|
152
|
+
const hasUntil = Option.isSome(until);
|
|
153
|
+
|
|
154
|
+
if (hasRange && (hasSince || hasUntil)) {
|
|
155
|
+
return yield* CliInputError.make({
|
|
156
|
+
message: "Use either --range or --since/--until, not both.",
|
|
157
|
+
cause: { range: range.value, since: Option.getOrUndefined(since), until: Option.getOrUndefined(until) }
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
if (hasRange) {
|
|
162
|
+
const parsed = yield* parseRange(range.value);
|
|
163
|
+
return Option.some(parsed);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
if (!hasSince && !hasUntil) {
|
|
167
|
+
return Option.none();
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const nowMillis = yield* Clock.currentTimeMillis;
|
|
171
|
+
const now = new Date(nowMillis);
|
|
172
|
+
|
|
173
|
+
const start = hasSince
|
|
174
|
+
? yield* parseTimeInput(since.value, now, { label: "--since" })
|
|
175
|
+
: new Date(0);
|
|
176
|
+
const end = hasUntil
|
|
177
|
+
? yield* parseTimeInput(until.value, now, { label: "--until" })
|
|
178
|
+
: now;
|
|
179
|
+
|
|
180
|
+
if (start.getTime() > end.getTime()) {
|
|
181
|
+
return yield* CliInputError.make({
|
|
182
|
+
message: `Invalid time range: start ${start.toISOString()} must be before end ${end.toISOString()}.`,
|
|
183
|
+
cause: { start, end }
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const startTimestamp = yield* toTimestamp(start, "start");
|
|
188
|
+
const endTimestamp = yield* toTimestamp(end, "end");
|
|
189
|
+
return Option.some({ start: startTimestamp, end: endTimestamp });
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
const splitStoreNames = (raw: ReadonlyArray<string>) =>
|
|
193
|
+
raw.flatMap((value) =>
|
|
194
|
+
value
|
|
195
|
+
.split(",")
|
|
196
|
+
.map((entry) => entry.trim())
|
|
197
|
+
.filter((entry) => entry.length > 0)
|
|
198
|
+
);
|
|
199
|
+
|
|
200
|
+
const parseStoreNames = (raw: ReadonlyArray<string>) =>
|
|
201
|
+
Effect.gen(function* () {
|
|
202
|
+
const names = splitStoreNames(raw);
|
|
203
|
+
if (names.length === 0) {
|
|
204
|
+
return yield* CliInputError.make({
|
|
205
|
+
message: "Provide at least one store name.",
|
|
206
|
+
cause: { stores: raw }
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
return yield* Effect.forEach(
|
|
210
|
+
names,
|
|
211
|
+
(name) =>
|
|
212
|
+
Schema.decodeUnknown(StoreName)(name).pipe(
|
|
213
|
+
Effect.mapError((error) =>
|
|
214
|
+
CliInputError.make({
|
|
215
|
+
message: `Invalid store name "${name}": ${formatSchemaError(error)}`,
|
|
216
|
+
cause: { name }
|
|
217
|
+
})
|
|
218
|
+
)
|
|
219
|
+
),
|
|
220
|
+
{ discard: false }
|
|
221
|
+
);
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
const loadStoreRefs = (names: ReadonlyArray<StoreName>) =>
|
|
225
|
+
Effect.gen(function* () {
|
|
226
|
+
const manager = yield* StoreManager;
|
|
227
|
+
const results = yield* Effect.forEach(
|
|
228
|
+
names,
|
|
229
|
+
(name) => manager.getStore(name),
|
|
230
|
+
{ discard: false }
|
|
231
|
+
);
|
|
232
|
+
const missing = names.filter((_, index) => Option.isNone(results[index]!));
|
|
233
|
+
if (missing.length > 0) {
|
|
234
|
+
if (missing.length === 1 && names.length === 1) {
|
|
235
|
+
return yield* StoreNotFound.make({ name: missing[0]! });
|
|
236
|
+
}
|
|
237
|
+
return yield* CliInputError.make({
|
|
238
|
+
message: `Unknown stores: ${missing.join(", ")}`,
|
|
239
|
+
cause: { missing }
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
const stores = results
|
|
243
|
+
.map((option) => (Option.isSome(option) ? option.value : undefined))
|
|
244
|
+
.filter((value): value is NonNullable<typeof value> => value !== undefined);
|
|
245
|
+
return stores;
|
|
108
246
|
});
|
|
109
247
|
|
|
110
248
|
|
|
111
249
|
export const queryCommand = Command.make(
|
|
112
250
|
"query",
|
|
113
251
|
{
|
|
114
|
-
|
|
252
|
+
stores: storeNamesArg,
|
|
115
253
|
range: rangeOption,
|
|
254
|
+
since: sinceOption,
|
|
255
|
+
until: untilOption,
|
|
116
256
|
filter: filterOption,
|
|
117
257
|
filterJson: filterJsonOption,
|
|
118
258
|
limit: limitOption,
|
|
@@ -120,51 +260,62 @@ export const queryCommand = Command.make(
|
|
|
120
260
|
sort: sortOption,
|
|
121
261
|
newestFirst: newestFirstOption,
|
|
122
262
|
format: formatOption,
|
|
263
|
+
includeStore: includeStoreOption,
|
|
123
264
|
ansi: ansiOption,
|
|
124
265
|
width: widthOption,
|
|
125
266
|
fields: fieldsOption,
|
|
126
|
-
progress: progressOption
|
|
267
|
+
progress: progressOption,
|
|
268
|
+
count: countOption
|
|
127
269
|
},
|
|
128
|
-
({
|
|
270
|
+
({ stores, range, since, until, filter, filterJson, limit, scanLimit, sort, newestFirst, format, includeStore, ansi, width, fields, progress, count }) =>
|
|
129
271
|
Effect.gen(function* () {
|
|
130
272
|
const appConfig = yield* AppConfigService;
|
|
131
273
|
const index = yield* StoreIndex;
|
|
132
274
|
const runtime = yield* FilterRuntime;
|
|
133
275
|
const output = yield* CliOutput;
|
|
134
276
|
const preferences = yield* CliPreferences;
|
|
135
|
-
const
|
|
136
|
-
const
|
|
277
|
+
const storeNames = yield* parseStoreNames(stores);
|
|
278
|
+
const storeRefs = yield* loadStoreRefs(storeNames);
|
|
279
|
+
const multiStore = storeRefs.length > 1;
|
|
280
|
+
const includeStoreLabel = includeStore || multiStore;
|
|
281
|
+
const parsedRange = yield* parseRangeOptions(range, since, until);
|
|
137
282
|
const parsedFilter = yield* parseOptionalFilterExpr(filter, filterJson);
|
|
138
283
|
const expr = Option.getOrElse(parsedFilter, () => all());
|
|
139
|
-
const outputFormat =
|
|
284
|
+
const outputFormat = resolveOutputFormat(
|
|
285
|
+
format,
|
|
286
|
+
appConfig.outputFormat,
|
|
287
|
+
queryOutputFormats,
|
|
288
|
+
"json"
|
|
289
|
+
);
|
|
290
|
+
if (multiStore && outputFormat === "thread") {
|
|
291
|
+
return yield* CliInputError.make({
|
|
292
|
+
message: "Thread output is only supported for single-store queries.",
|
|
293
|
+
cause: { format: outputFormat }
|
|
294
|
+
});
|
|
295
|
+
}
|
|
140
296
|
const compact = preferences.compact;
|
|
141
|
-
const selectorsOption
|
|
297
|
+
const { selectors: selectorsOption, source: selectorsSource } =
|
|
298
|
+
yield* resolveFieldSelectors(fields, compact);
|
|
142
299
|
const project = (post: Post) =>
|
|
143
300
|
Option.match(selectorsOption, {
|
|
144
301
|
onNone: () => post,
|
|
145
302
|
onSome: (selectors) => projectFields(post, selectors)
|
|
146
303
|
});
|
|
147
|
-
if (
|
|
304
|
+
if (selectorsSource === "explicit" && outputFormat !== "json" && outputFormat !== "ndjson") {
|
|
148
305
|
return yield* CliInputError.make({
|
|
149
306
|
message: "--fields is only supported with json or ndjson output.",
|
|
150
307
|
cause: { format: outputFormat }
|
|
151
308
|
});
|
|
152
309
|
}
|
|
153
|
-
|
|
154
|
-
const w = Option.getOrUndefined(width);
|
|
155
|
-
|
|
156
|
-
if (Option.isSome(limit) && limit.value <= 0) {
|
|
310
|
+
if (count && selectorsSource === "explicit") {
|
|
157
311
|
return yield* CliInputError.make({
|
|
158
|
-
message: "--
|
|
159
|
-
cause: {
|
|
160
|
-
});
|
|
161
|
-
}
|
|
162
|
-
if (Option.isSome(scanLimit) && scanLimit.value <= 0) {
|
|
163
|
-
return yield* CliInputError.make({
|
|
164
|
-
message: "--scan-limit must be a positive integer.",
|
|
165
|
-
cause: { scanLimit: scanLimit.value }
|
|
312
|
+
message: "--count cannot be combined with --fields.",
|
|
313
|
+
cause: { count, fields }
|
|
166
314
|
});
|
|
167
315
|
}
|
|
316
|
+
|
|
317
|
+
const w = Option.getOrUndefined(width);
|
|
318
|
+
|
|
168
319
|
const sortValue = Option.getOrUndefined(sort);
|
|
169
320
|
const order =
|
|
170
321
|
newestFirst
|
|
@@ -198,19 +349,18 @@ export const queryCommand = Command.make(
|
|
|
198
349
|
if (defaultScanLimit !== undefined) {
|
|
199
350
|
yield* output
|
|
200
351
|
.writeStderr(
|
|
201
|
-
`ℹ️ Scanning up to ${defaultScanLimit} posts (filtered query). Use --scan-limit to scan more.`
|
|
352
|
+
`ℹ️ Scanning up to ${defaultScanLimit} posts${multiStore ? " per store" : ""} (filtered query). Use --scan-limit to scan more.`
|
|
202
353
|
)
|
|
203
354
|
.pipe(Effect.catchAll(() => Effect.void));
|
|
204
355
|
}
|
|
205
356
|
|
|
206
357
|
if (
|
|
207
|
-
hasFilter &&
|
|
208
358
|
Option.isNone(limit) &&
|
|
209
|
-
(outputFormat === "thread" || outputFormat === "table")
|
|
359
|
+
(outputFormat === "thread" || outputFormat === "table" || outputFormat === "markdown")
|
|
210
360
|
) {
|
|
211
361
|
yield* output
|
|
212
362
|
.writeStderr(
|
|
213
|
-
"Warning: thread
|
|
363
|
+
"Warning: table/markdown/thread output collects all matched posts in memory. Consider adding --limit."
|
|
214
364
|
)
|
|
215
365
|
.pipe(Effect.catchAll(() => Effect.void));
|
|
216
366
|
}
|
|
@@ -222,12 +372,8 @@ export const queryCommand = Command.make(
|
|
|
222
372
|
order
|
|
223
373
|
});
|
|
224
374
|
|
|
225
|
-
const baseStream = index.query(storeRef, query);
|
|
226
375
|
const progressEnabled = hasFilter && progress;
|
|
227
376
|
const trackScanLimit = hasFilter && resolvedScanLimit !== undefined;
|
|
228
|
-
const scanRef = trackScanLimit
|
|
229
|
-
? yield* Ref.make({ scanned: 0, matched: 0 })
|
|
230
|
-
: undefined;
|
|
231
377
|
let startTime = 0;
|
|
232
378
|
let progressRef: Ref.Ref<{ scanned: number; matched: number; lastReportAt: number }> | undefined;
|
|
233
379
|
if (progressEnabled) {
|
|
@@ -245,7 +391,11 @@ export const queryCommand = Command.make(
|
|
|
245
391
|
.pipe(Effect.catchAll(() => Effect.void))
|
|
246
392
|
: undefined;
|
|
247
393
|
|
|
248
|
-
const onBatch = (
|
|
394
|
+
const onBatch = (
|
|
395
|
+
scanRef: Ref.Ref<{ scanned: number; matched: number }> | undefined,
|
|
396
|
+
scannedDelta: number,
|
|
397
|
+
matchedDelta: number
|
|
398
|
+
) =>
|
|
249
399
|
Effect.gen(function* () {
|
|
250
400
|
if (scanRef) {
|
|
251
401
|
yield* Ref.update(scanRef, (state) => ({
|
|
@@ -273,49 +423,110 @@ export const queryCommand = Command.make(
|
|
|
273
423
|
|
|
274
424
|
const evaluateBatch = hasFilter ? yield* runtime.evaluateBatch(expr) : undefined;
|
|
275
425
|
|
|
276
|
-
const
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
426
|
+
const buildStoreStream = (storeRef: StoreRef) =>
|
|
427
|
+
Effect.gen(function* () {
|
|
428
|
+
const scanRef = trackScanLimit
|
|
429
|
+
? yield* Ref.make({ scanned: 0, matched: 0 })
|
|
430
|
+
: undefined;
|
|
431
|
+
const baseStream = index.query(storeRef, query);
|
|
432
|
+
const filtered = hasFilter && evaluateBatch
|
|
433
|
+
? baseStream.pipe(
|
|
434
|
+
Stream.grouped(50),
|
|
435
|
+
Stream.mapEffect((batch) =>
|
|
436
|
+
evaluateBatch(batch).pipe(
|
|
437
|
+
Effect.map((flags) => {
|
|
438
|
+
const matched = filterByFlags(batch, flags);
|
|
439
|
+
return {
|
|
440
|
+
matched,
|
|
441
|
+
scanned: Chunk.size(batch),
|
|
442
|
+
matchedCount: Chunk.size(matched)
|
|
443
|
+
};
|
|
444
|
+
}),
|
|
445
|
+
Effect.tap(({ scanned, matchedCount }) =>
|
|
446
|
+
onBatch(scanRef, scanned, matchedCount)
|
|
447
|
+
),
|
|
448
|
+
Effect.map(({ matched }) => matched)
|
|
449
|
+
)
|
|
450
|
+
),
|
|
451
|
+
Stream.mapConcat((chunk) => Chunk.toReadonlyArray(chunk))
|
|
291
452
|
)
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
453
|
+
: baseStream;
|
|
454
|
+
const storeStream = filtered.pipe(
|
|
455
|
+
Stream.map((post) => ({ store: storeRef, post }))
|
|
456
|
+
);
|
|
457
|
+
return { store: storeRef, stream: storeStream, scanRef };
|
|
458
|
+
});
|
|
459
|
+
|
|
460
|
+
const storeStreams = yield* Effect.forEach(storeRefs, buildStoreStream, {
|
|
461
|
+
discard: false
|
|
462
|
+
});
|
|
463
|
+
const scanRefs = storeStreams
|
|
464
|
+
.map((entry) =>
|
|
465
|
+
entry.scanRef ? { store: entry.store, ref: entry.scanRef } : undefined
|
|
466
|
+
)
|
|
467
|
+
.filter((entry): entry is { store: StoreRef; ref: Ref.Ref<{ scanned: number; matched: number }> } => entry !== undefined);
|
|
468
|
+
|
|
469
|
+
const storePostOrder =
|
|
470
|
+
order === "desc" ? Order.reverse(StorePostOrder) : StorePostOrder;
|
|
471
|
+
|
|
472
|
+
const merged = mergeOrderedStreams(
|
|
473
|
+
storeStreams.map((entry) => entry.stream),
|
|
474
|
+
storePostOrder
|
|
475
|
+
);
|
|
296
476
|
|
|
297
477
|
const stream = Option.match(limit, {
|
|
298
|
-
onNone: () =>
|
|
299
|
-
onSome: (value) =>
|
|
478
|
+
onNone: () => merged,
|
|
479
|
+
onSome: (value) => merged.pipe(Stream.take(value))
|
|
300
480
|
});
|
|
301
481
|
|
|
302
|
-
const warnIfScanLimitReached =
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
482
|
+
const warnIfScanLimitReached = () =>
|
|
483
|
+
resolvedScanLimit === undefined || scanRefs.length === 0
|
|
484
|
+
? Effect.void
|
|
485
|
+
: Effect.forEach(
|
|
486
|
+
scanRefs,
|
|
487
|
+
({ store, ref }) =>
|
|
488
|
+
Ref.get(ref).pipe(
|
|
489
|
+
Effect.flatMap((state) =>
|
|
490
|
+
state.scanned >= resolvedScanLimit
|
|
491
|
+
? output
|
|
492
|
+
.writeStderr(
|
|
493
|
+
multiStore
|
|
494
|
+
? `Warning: scan limit ${resolvedScanLimit} reached for ${store.name}. Results may be truncated.\n`
|
|
495
|
+
: `Warning: scan limit ${resolvedScanLimit} reached. Results may be truncated.\n`
|
|
496
|
+
)
|
|
497
|
+
.pipe(Effect.catchAll(() => Effect.void))
|
|
498
|
+
: Effect.void
|
|
499
|
+
)
|
|
500
|
+
),
|
|
501
|
+
{ discard: true }
|
|
502
|
+
);
|
|
503
|
+
|
|
504
|
+
const toOutput = (entry: StorePost) => {
|
|
505
|
+
const projected = project(entry.post);
|
|
506
|
+
return includeStoreLabel ? { store: entry.store.name, post: projected } : projected;
|
|
507
|
+
};
|
|
508
|
+
|
|
509
|
+
if (count) {
|
|
510
|
+
const canUseIndexCount =
|
|
511
|
+
!hasFilter && Option.isNone(parsedRange);
|
|
512
|
+
const total = canUseIndexCount
|
|
513
|
+
? yield* Effect.forEach(storeRefs, (store) => index.count(store), {
|
|
514
|
+
discard: false
|
|
515
|
+
}).pipe(
|
|
516
|
+
Effect.map((counts) => counts.reduce((sum, value) => sum + value, 0))
|
|
314
517
|
)
|
|
315
|
-
|
|
518
|
+
: yield* Stream.runFold(stream, 0, (acc) => acc + 1);
|
|
519
|
+
const limited = Option.match(limit, {
|
|
520
|
+
onNone: () => total,
|
|
521
|
+
onSome: (value) => Math.min(total, value)
|
|
522
|
+
});
|
|
523
|
+
yield* writeJson(limited);
|
|
524
|
+
yield* warnIfScanLimitReached();
|
|
525
|
+
return;
|
|
526
|
+
}
|
|
316
527
|
|
|
317
528
|
if (outputFormat === "ndjson") {
|
|
318
|
-
yield* writeJsonStream(stream.pipe(Stream.map(
|
|
529
|
+
yield* writeJsonStream(stream.pipe(Stream.map(toOutput)));
|
|
319
530
|
yield* warnIfScanLimitReached();
|
|
320
531
|
return;
|
|
321
532
|
}
|
|
@@ -324,8 +535,8 @@ export const queryCommand = Command.make(
|
|
|
324
535
|
Stream.fromIterable([value]).pipe(Stream.run(output.stdout));
|
|
325
536
|
let isFirst = true;
|
|
326
537
|
yield* writeChunk("[");
|
|
327
|
-
yield* Stream.runForEach(stream.pipe(Stream.map(
|
|
328
|
-
const json = JSON.stringify(
|
|
538
|
+
yield* Stream.runForEach(stream.pipe(Stream.map(toOutput)), (value) => {
|
|
539
|
+
const json = JSON.stringify(value);
|
|
329
540
|
const prefix = isFirst ? "" : ",\n";
|
|
330
541
|
isFirst = false;
|
|
331
542
|
return writeChunk(`${prefix}${json}`);
|
|
@@ -338,44 +549,83 @@ export const queryCommand = Command.make(
|
|
|
338
549
|
|
|
339
550
|
switch (outputFormat) {
|
|
340
551
|
case "compact": {
|
|
341
|
-
const
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
552
|
+
const countRef = yield* Ref.make(0);
|
|
553
|
+
const render = (entry: StorePost) => {
|
|
554
|
+
const doc = includeStoreLabel
|
|
555
|
+
? Doc.hsep([Doc.text(`[${entry.store.name}]`), renderPostCompact(entry.post)])
|
|
556
|
+
: renderPostCompact(entry.post);
|
|
557
|
+
return ansi ? renderAnsi(doc, w) : renderPlain(doc, w);
|
|
558
|
+
};
|
|
559
|
+
yield* Stream.runForEach(stream, (entry) =>
|
|
560
|
+
Ref.update(countRef, (count) => count + 1).pipe(
|
|
561
|
+
Effect.zipRight(writeText(render(entry)))
|
|
562
|
+
)
|
|
563
|
+
);
|
|
564
|
+
const count = yield* Ref.get(countRef);
|
|
565
|
+
if (count === 0) {
|
|
566
|
+
yield* writeText("No posts found.");
|
|
567
|
+
}
|
|
346
568
|
yield* warnIfScanLimitReached();
|
|
347
569
|
return;
|
|
348
570
|
}
|
|
349
571
|
case "card": {
|
|
572
|
+
const countRef = yield* Ref.make(0);
|
|
350
573
|
const rendered = stream.pipe(
|
|
351
|
-
Stream.map((
|
|
352
|
-
const
|
|
574
|
+
Stream.map((entry) => {
|
|
575
|
+
const lines = renderPostCard(entry.post);
|
|
576
|
+
const doc = includeStoreLabel
|
|
577
|
+
? Doc.vsep([Doc.text(`[${entry.store.name}]`), ...lines])
|
|
578
|
+
: Doc.vsep(lines);
|
|
353
579
|
return ansi ? renderAnsi(doc, w) : renderPlain(doc, w);
|
|
354
580
|
}),
|
|
355
581
|
Stream.mapAccum(true, (isFirst, text) => {
|
|
356
582
|
const output = isFirst ? text : `\\n${text}`;
|
|
357
583
|
return [false, output] as const;
|
|
358
|
-
})
|
|
584
|
+
}),
|
|
585
|
+
Stream.tap(() => Ref.update(countRef, (count) => count + 1))
|
|
359
586
|
);
|
|
360
587
|
yield* Stream.runForEach(rendered, (text) => writeText(text));
|
|
588
|
+
const count = yield* Ref.get(countRef);
|
|
589
|
+
if (count === 0) {
|
|
590
|
+
yield* writeText("No posts found.");
|
|
591
|
+
}
|
|
361
592
|
yield* warnIfScanLimitReached();
|
|
362
593
|
return;
|
|
363
594
|
}
|
|
364
595
|
}
|
|
365
596
|
|
|
366
597
|
const collected = yield* Stream.runCollect(stream);
|
|
367
|
-
const
|
|
368
|
-
const
|
|
598
|
+
const entries = Chunk.toReadonlyArray(collected);
|
|
599
|
+
const posts = entries.map((entry) => entry.post);
|
|
600
|
+
const projectedPosts = entries.map(toOutput);
|
|
369
601
|
yield* warnIfScanLimitReached();
|
|
370
602
|
|
|
371
603
|
switch (outputFormat) {
|
|
372
604
|
case "markdown":
|
|
373
|
-
yield* writeText(
|
|
605
|
+
yield* writeText(
|
|
606
|
+
includeStoreLabel
|
|
607
|
+
? renderStorePostsMarkdown(entries.map((entry) => ({
|
|
608
|
+
store: entry.store.name,
|
|
609
|
+
post: entry.post
|
|
610
|
+
})))
|
|
611
|
+
: renderPostsMarkdown(posts)
|
|
612
|
+
);
|
|
374
613
|
return;
|
|
375
614
|
case "table":
|
|
376
|
-
yield* writeText(
|
|
615
|
+
yield* writeText(
|
|
616
|
+
includeStoreLabel
|
|
617
|
+
? renderStorePostsTable(entries.map((entry) => ({
|
|
618
|
+
store: entry.store.name,
|
|
619
|
+
post: entry.post
|
|
620
|
+
})))
|
|
621
|
+
: renderPostsTable(posts)
|
|
622
|
+
);
|
|
377
623
|
return;
|
|
378
624
|
case "thread": {
|
|
625
|
+
if (posts.length === 0) {
|
|
626
|
+
yield* writeText("No posts found.");
|
|
627
|
+
return;
|
|
628
|
+
}
|
|
379
629
|
// B3: Warn if query doesn't have thread relationships
|
|
380
630
|
if (!hasFilter) {
|
|
381
631
|
yield* output
|
|
@@ -398,14 +648,18 @@ export const queryCommand = Command.make(
|
|
|
398
648
|
).pipe(
|
|
399
649
|
Command.withDescription(
|
|
400
650
|
withExamples(
|
|
401
|
-
"Query a store with optional range and filter",
|
|
651
|
+
"Query a store with optional time range and filter",
|
|
402
652
|
[
|
|
403
653
|
"skygent query my-store --limit 25 --format table",
|
|
404
654
|
"skygent query my-store --range 2024-01-01T00:00:00Z..2024-01-31T00:00:00Z --filter 'hashtag:#ai'",
|
|
655
|
+
"skygent query my-store --since 24h --filter 'hashtag:#ai'",
|
|
656
|
+
"skygent query my-store --until 2024-01-15 --format compact",
|
|
405
657
|
"skygent query my-store --format card --ansi",
|
|
406
658
|
"skygent query my-store --format thread --ansi --width 120",
|
|
407
659
|
"skygent query my-store --format compact --limit 50",
|
|
408
|
-
"skygent query my-store --sort desc --limit 25"
|
|
660
|
+
"skygent query my-store --sort desc --limit 25",
|
|
661
|
+
"skygent query my-store --filter 'contains:ai' --count",
|
|
662
|
+
"skygent query store-a,store-b --format ndjson"
|
|
409
663
|
],
|
|
410
664
|
[
|
|
411
665
|
"Tip: use --fields @minimal or --compact to reduce JSON output size."
|