@mepuka/skygent 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +269 -31
- package/index.ts +18 -3
- package/package.json +1 -1
- package/src/cli/app.ts +4 -2
- package/src/cli/compact-output.ts +52 -0
- package/src/cli/config.ts +46 -4
- package/src/cli/doc/table-renderers.ts +29 -0
- package/src/cli/doc/thread.ts +2 -4
- package/src/cli/exit-codes.ts +2 -0
- package/src/cli/feed.ts +78 -61
- package/src/cli/filter-dsl.ts +146 -11
- package/src/cli/filter-errors.ts +13 -11
- package/src/cli/filter-help.ts +7 -0
- package/src/cli/filter-input.ts +3 -2
- package/src/cli/filter.ts +83 -5
- package/src/cli/graph.ts +297 -169
- package/src/cli/input.ts +45 -0
- package/src/cli/interval.ts +4 -33
- package/src/cli/jetstream.ts +2 -0
- package/src/cli/layers.ts +10 -0
- package/src/cli/logging.ts +8 -0
- package/src/cli/option-schemas.ts +22 -0
- package/src/cli/output-format.ts +11 -0
- package/src/cli/output-render.ts +14 -0
- package/src/cli/pagination.ts +17 -0
- package/src/cli/parse-errors.ts +30 -0
- package/src/cli/parse.ts +1 -47
- package/src/cli/pipe-input.ts +18 -0
- package/src/cli/pipe.ts +154 -0
- package/src/cli/post.ts +88 -66
- package/src/cli/query-fields.ts +13 -3
- package/src/cli/query.ts +354 -100
- package/src/cli/search.ts +93 -136
- package/src/cli/shared-options.ts +11 -63
- package/src/cli/shared.ts +1 -20
- package/src/cli/store-errors.ts +28 -21
- package/src/cli/store-tree.ts +6 -4
- package/src/cli/store.ts +41 -2
- package/src/cli/stream-merge.ts +105 -0
- package/src/cli/sync-factory.ts +24 -7
- package/src/cli/sync.ts +46 -67
- package/src/cli/thread-options.ts +25 -0
- package/src/cli/time.ts +171 -0
- package/src/cli/view-thread.ts +29 -32
- package/src/cli/watch.ts +55 -26
- package/src/domain/errors.ts +6 -1
- package/src/domain/format.ts +21 -0
- package/src/domain/order.ts +24 -0
- package/src/domain/primitives.ts +20 -3
- package/src/graph/relationships.ts +129 -0
- package/src/services/bsky-client.ts +11 -5
- package/src/services/jetstream-sync.ts +4 -4
- package/src/services/lineage-store.ts +15 -1
- package/src/services/shared.ts +48 -1
- package/src/services/store-cleaner.ts +5 -2
- package/src/services/store-commit.ts +60 -0
- package/src/services/store-manager.ts +69 -2
- package/src/services/store-renamer.ts +288 -0
- package/src/services/store-stats.ts +7 -5
- package/src/services/sync-engine.ts +149 -89
- package/src/services/sync-reporter.ts +3 -1
- package/src/services/sync-settings.ts +24 -0
package/src/cli/interval.ts
CHANGED
|
@@ -1,35 +1,6 @@
|
|
|
1
|
-
import { Duration,
|
|
2
|
-
import { CliInputError } from "./errors.js";
|
|
1
|
+
import { Duration, Option } from "effect";
|
|
3
2
|
|
|
4
|
-
const
|
|
5
|
-
|
|
6
|
-
try: () => Duration.decode(value as Duration.DurationInput),
|
|
7
|
-
catch: (cause) =>
|
|
8
|
-
CliInputError.make({
|
|
9
|
-
message: `Invalid duration: ${value}. Use formats like "30 seconds" or "500 millis".`,
|
|
10
|
-
cause
|
|
11
|
-
})
|
|
12
|
-
}).pipe(
|
|
13
|
-
Effect.flatMap((duration) =>
|
|
14
|
-
Duration.toMillis(duration) < 0
|
|
15
|
-
? Effect.fail(
|
|
16
|
-
CliInputError.make({
|
|
17
|
-
message: "Interval must be non-negative.",
|
|
18
|
-
cause: duration
|
|
19
|
-
})
|
|
20
|
-
)
|
|
21
|
-
: Effect.succeed(duration)
|
|
22
|
-
)
|
|
23
|
-
);
|
|
3
|
+
export const parseInterval = (interval: Option.Option<Duration.Duration>) =>
|
|
4
|
+
Option.getOrElse(interval, () => Duration.seconds(30));
|
|
24
5
|
|
|
25
|
-
export const
|
|
26
|
-
Option.match(interval, {
|
|
27
|
-
onSome: parseDurationText,
|
|
28
|
-
onNone: () => Effect.succeed(Duration.seconds(30))
|
|
29
|
-
});
|
|
30
|
-
|
|
31
|
-
export const parseOptionalDuration = (value: Option.Option<string>) =>
|
|
32
|
-
Option.match(value, {
|
|
33
|
-
onSome: (raw) => parseDurationText(raw).pipe(Effect.map(Option.some)),
|
|
34
|
-
onNone: () => Effect.succeed(Option.none())
|
|
35
|
-
});
|
|
6
|
+
export const parseOptionalDuration = (value: Option.Option<Duration.Duration>) => value;
|
package/src/cli/jetstream.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { DataSource } from "../domain/sync.js";
|
|
|
5
5
|
import type { StoreRef } from "../domain/store.js";
|
|
6
6
|
import { SyncCheckpointStore } from "../services/sync-checkpoint-store.js";
|
|
7
7
|
import { CliInputError } from "./errors.js";
|
|
8
|
+
import { PositiveInt } from "./option-schemas.js";
|
|
8
9
|
|
|
9
10
|
const DEFAULT_COLLECTIONS = ["app.bsky.feed.post"];
|
|
10
11
|
|
|
@@ -31,6 +32,7 @@ export const jetstreamOptions = {
|
|
|
31
32
|
Options.withDescription("Enable compression if supported by runtime")
|
|
32
33
|
),
|
|
33
34
|
maxMessageSize: Options.integer("max-message-size").pipe(
|
|
35
|
+
Options.withSchema(PositiveInt),
|
|
34
36
|
Options.withDescription("Max message size in bytes"),
|
|
35
37
|
Options.optional
|
|
36
38
|
)
|
package/src/cli/layers.ts
CHANGED
|
@@ -18,10 +18,12 @@ import { SyncCheckpointStore } from "../services/sync-checkpoint-store.js";
|
|
|
18
18
|
import { SyncReporter } from "../services/sync-reporter.js";
|
|
19
19
|
import { SyncSettings } from "../services/sync-settings.js";
|
|
20
20
|
import { StoreCleaner } from "../services/store-cleaner.js";
|
|
21
|
+
import { StoreRenamer } from "../services/store-renamer.js";
|
|
21
22
|
import { LinkValidator } from "../services/link-validator.js";
|
|
22
23
|
import { TrendingTopics } from "../services/trending-topics.js";
|
|
23
24
|
import { ResourceMonitor } from "../services/resource-monitor.js";
|
|
24
25
|
import { CliOutput } from "./output.js";
|
|
26
|
+
import { CliInput } from "./input.js";
|
|
25
27
|
import { DerivationEngine } from "../services/derivation-engine.js";
|
|
26
28
|
import { DerivationValidator } from "../services/derivation-validator.js";
|
|
27
29
|
import { DerivationSettings } from "../services/derivation-settings.js";
|
|
@@ -114,6 +116,12 @@ const viewCheckpointLayer = ViewCheckpointStore.layer.pipe(
|
|
|
114
116
|
const lineageLayer = LineageStore.layer.pipe(
|
|
115
117
|
Layer.provideMerge(storageLayer)
|
|
116
118
|
);
|
|
119
|
+
const storeRenamerLayer = StoreRenamer.layer.pipe(
|
|
120
|
+
Layer.provideMerge(appConfigLayer),
|
|
121
|
+
Layer.provideMerge(managerLayer),
|
|
122
|
+
Layer.provideMerge(storeDbLayer),
|
|
123
|
+
Layer.provideMerge(lineageLayer)
|
|
124
|
+
);
|
|
117
125
|
const compilerLayer = FilterCompiler.layer;
|
|
118
126
|
const postParserLayer = PostParser.layer;
|
|
119
127
|
const derivationEngineLayer = DerivationEngine.layer.pipe(
|
|
@@ -157,6 +165,7 @@ export const CliLive = Layer.mergeAll(
|
|
|
157
165
|
appConfigLayer,
|
|
158
166
|
filterSettingsLayer,
|
|
159
167
|
credentialLayer,
|
|
168
|
+
CliInput.layer,
|
|
160
169
|
CliOutput.layer,
|
|
161
170
|
resourceMonitorLayer,
|
|
162
171
|
managerLayer,
|
|
@@ -164,6 +173,7 @@ export const CliLive = Layer.mergeAll(
|
|
|
164
173
|
indexLayer,
|
|
165
174
|
eventLogLayer,
|
|
166
175
|
cleanerLayer,
|
|
176
|
+
storeRenamerLayer,
|
|
167
177
|
syncLayer,
|
|
168
178
|
checkpointLayer,
|
|
169
179
|
viewCheckpointLayer,
|
package/src/cli/logging.ts
CHANGED
|
@@ -132,5 +132,13 @@ export const makeSyncReporter = (
|
|
|
132
132
|
{ discard: true }
|
|
133
133
|
);
|
|
134
134
|
}
|
|
135
|
+
}).pipe(Effect.orElseSucceed(() => undefined)),
|
|
136
|
+
warn: (message, data) =>
|
|
137
|
+
Effect.gen(function* () {
|
|
138
|
+
const format = yield* resolveLogFormat;
|
|
139
|
+
yield* logEventWith(output, format, "WARN", {
|
|
140
|
+
message,
|
|
141
|
+
...data
|
|
142
|
+
});
|
|
135
143
|
}).pipe(Effect.orElseSucceed(() => undefined))
|
|
136
144
|
});
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { Duration, Effect, ParseResult, Schema } from "effect";
|
|
2
|
+
import { parseDurationInput } from "./time.js";
|
|
3
|
+
|
|
4
|
+
export const PositiveInt = Schema.Int.pipe(Schema.greaterThan(0));
|
|
5
|
+
|
|
6
|
+
export const NonNegativeInt = Schema.NonNegativeInt;
|
|
7
|
+
|
|
8
|
+
export const boundedInt = (min: number, max: number) =>
|
|
9
|
+
Schema.Int.pipe(
|
|
10
|
+
Schema.greaterThanOrEqualTo(min),
|
|
11
|
+
Schema.lessThanOrEqualTo(max)
|
|
12
|
+
);
|
|
13
|
+
|
|
14
|
+
export const DurationInput = Schema.transformOrFail(Schema.String, Schema.DurationFromSelf, {
|
|
15
|
+
strict: true,
|
|
16
|
+
decode: (raw, _options, ast) =>
|
|
17
|
+
parseDurationInput(raw).pipe(
|
|
18
|
+
Effect.mapError((error) => new ParseResult.Type(ast, raw, error.message))
|
|
19
|
+
),
|
|
20
|
+
encode: (duration) =>
|
|
21
|
+
Effect.succeed(`${Duration.toMillis(duration)} millis`)
|
|
22
|
+
}).pipe(Schema.greaterThanOrEqualToDuration(0));
|
package/src/cli/output-format.ts
CHANGED
|
@@ -13,6 +13,17 @@ export type TextJsonFormat = typeof textJsonFormats[number];
|
|
|
13
13
|
export const treeTableJsonFormats = ["tree", "table", "json"] as const;
|
|
14
14
|
export type TreeTableJsonFormat = typeof treeTableJsonFormats[number];
|
|
15
15
|
|
|
16
|
+
export const queryOutputFormats = [
|
|
17
|
+
"json",
|
|
18
|
+
"ndjson",
|
|
19
|
+
"markdown",
|
|
20
|
+
"table",
|
|
21
|
+
"compact",
|
|
22
|
+
"card",
|
|
23
|
+
"thread"
|
|
24
|
+
] as const;
|
|
25
|
+
export type QueryOutputFormat = typeof queryOutputFormats[number];
|
|
26
|
+
|
|
16
27
|
export const resolveOutputFormat = <T extends string>(
|
|
17
28
|
format: Option.Option<T>,
|
|
18
29
|
configFormat: OutputFormat,
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Effect, Option } from "effect";
|
|
2
|
+
import type { OutputFormat } from "../domain/config.js";
|
|
3
|
+
import { resolveOutputFormat } from "./output-format.js";
|
|
4
|
+
|
|
5
|
+
export const emitWithFormat = <T extends string, E, R>(
|
|
6
|
+
format: Option.Option<T>,
|
|
7
|
+
configFormat: OutputFormat,
|
|
8
|
+
supported: readonly T[],
|
|
9
|
+
fallback: T,
|
|
10
|
+
handlers: { readonly [K in T]: Effect.Effect<unknown, E, R> }
|
|
11
|
+
): Effect.Effect<unknown, E, R> => {
|
|
12
|
+
const resolved = resolveOutputFormat(format, configFormat, supported, fallback);
|
|
13
|
+
return handlers[resolved];
|
|
14
|
+
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { Options } from "@effect/cli";
|
|
2
|
+
import { Option } from "effect";
|
|
3
|
+
import { PositiveInt } from "./option-schemas.js";
|
|
4
|
+
|
|
5
|
+
export const limitOption = Options.integer("limit").pipe(
|
|
6
|
+
Options.withSchema(PositiveInt),
|
|
7
|
+
Options.optional
|
|
8
|
+
);
|
|
9
|
+
export const cursorOption = Options.text("cursor").pipe(Options.optional);
|
|
10
|
+
|
|
11
|
+
export const parsePagination = (
|
|
12
|
+
limit: Option.Option<number>,
|
|
13
|
+
cursor: Option.Option<string>
|
|
14
|
+
) => ({
|
|
15
|
+
limit: Option.getOrUndefined(limit),
|
|
16
|
+
cursor: Option.getOrUndefined(cursor)
|
|
17
|
+
});
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
type ParseIssue = { readonly _tag: string; readonly message?: string };
|
|
2
|
+
|
|
3
|
+
/** Safely parse JSON, returning `undefined` on failure. */
|
|
4
|
+
export const safeParseJson = (raw: string): unknown => {
|
|
5
|
+
try {
|
|
6
|
+
return JSON.parse(raw);
|
|
7
|
+
} catch {
|
|
8
|
+
return undefined;
|
|
9
|
+
}
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
export const jsonParseTip = "Tip: wrap JSON in single quotes to avoid shell escaping issues.";
|
|
13
|
+
|
|
14
|
+
export const findJsonParseIssue = (issues: ReadonlyArray<ParseIssue>) =>
|
|
15
|
+
issues.find(
|
|
16
|
+
(issue) =>
|
|
17
|
+
issue._tag === "Transformation" &&
|
|
18
|
+
typeof issue.message === "string" &&
|
|
19
|
+
issue.message.startsWith("JSON Parse error")
|
|
20
|
+
);
|
|
21
|
+
|
|
22
|
+
/** Format schema issues into an array of "path: message" strings. */
|
|
23
|
+
export const issueDetails = (
|
|
24
|
+
issues: ReadonlyArray<{ readonly path: ReadonlyArray<unknown>; readonly message: string }>
|
|
25
|
+
) =>
|
|
26
|
+
issues.map((issue) => {
|
|
27
|
+
const path =
|
|
28
|
+
issue.path.length > 0 ? issue.path.map((entry) => String(entry)).join(".") : "value";
|
|
29
|
+
return `${path}: ${issue.message}`;
|
|
30
|
+
});
|
package/src/cli/parse.ts
CHANGED
|
@@ -1,52 +1,6 @@
|
|
|
1
1
|
import { Effect, ParseResult, Schema } from "effect";
|
|
2
2
|
import { CliJsonError } from "./errors.js";
|
|
3
|
-
|
|
4
|
-
type FormatParseErrorOptions = {
|
|
5
|
-
readonly label?: string;
|
|
6
|
-
readonly maxIssues?: number;
|
|
7
|
-
};
|
|
8
|
-
|
|
9
|
-
const formatPath = (path: ReadonlyArray<unknown>) =>
|
|
10
|
-
path.length > 0 ? path.map((entry) => String(entry)).join(".") : "value";
|
|
11
|
-
|
|
12
|
-
const formatParseError = (
|
|
13
|
-
error: ParseResult.ParseError,
|
|
14
|
-
options?: FormatParseErrorOptions
|
|
15
|
-
) => {
|
|
16
|
-
const issues = ParseResult.ArrayFormatter.formatErrorSync(error);
|
|
17
|
-
if (issues.length === 0) {
|
|
18
|
-
return ParseResult.TreeFormatter.formatErrorSync(error);
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
const jsonParseIssue = issues.find(
|
|
22
|
-
(issue) =>
|
|
23
|
-
issue._tag === "Transformation" &&
|
|
24
|
-
typeof issue.message === "string" &&
|
|
25
|
-
issue.message.startsWith("JSON Parse error")
|
|
26
|
-
);
|
|
27
|
-
if (jsonParseIssue) {
|
|
28
|
-
const header = options?.label
|
|
29
|
-
? `Invalid JSON input for ${options.label}.`
|
|
30
|
-
: "Invalid JSON input.";
|
|
31
|
-
return [
|
|
32
|
-
header,
|
|
33
|
-
jsonParseIssue.message,
|
|
34
|
-
"Tip: wrap JSON in single quotes to avoid shell escaping issues."
|
|
35
|
-
].join("\n");
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
const maxIssues = options?.maxIssues ?? 6;
|
|
39
|
-
const lines = issues.slice(0, maxIssues).map((issue) => {
|
|
40
|
-
const path = formatPath(issue.path);
|
|
41
|
-
return `${path}: ${issue.message}`;
|
|
42
|
-
});
|
|
43
|
-
if (issues.length > maxIssues) {
|
|
44
|
-
lines.push(`Additional issues: ${issues.length - maxIssues}`);
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
const header = options?.label ? `Invalid ${options.label}.` : undefined;
|
|
48
|
-
return header ? [header, ...lines].join("\n") : lines.join("\n");
|
|
49
|
-
};
|
|
3
|
+
import { formatParseError } from "./shared.js";
|
|
50
4
|
|
|
51
5
|
type DecodeJsonOptions = {
|
|
52
6
|
readonly formatter?: (error: ParseResult.ParseError, raw: string) => string;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { Schema } from "effect";
|
|
2
|
+
import { RawPost } from "../domain/raw.js";
|
|
3
|
+
import { Post } from "../domain/post.js";
|
|
4
|
+
import { StoreName } from "../domain/primitives.js";
|
|
5
|
+
|
|
6
|
+
export class StorePostInput extends Schema.Class<StorePostInput>("StorePostInput")({
|
|
7
|
+
store: StoreName,
|
|
8
|
+
post: Post
|
|
9
|
+
}) {}
|
|
10
|
+
|
|
11
|
+
export const PipeInput = Schema.Union(RawPost, Post, StorePostInput);
|
|
12
|
+
export type PipeInput = typeof PipeInput.Type;
|
|
13
|
+
|
|
14
|
+
export const isRawPostInput = (value: PipeInput): value is RawPost =>
|
|
15
|
+
typeof value === "object" && value !== null && "record" in value;
|
|
16
|
+
|
|
17
|
+
export const isStorePostInput = (value: PipeInput): value is StorePostInput =>
|
|
18
|
+
typeof value === "object" && value !== null && "post" in value;
|
package/src/cli/pipe.ts
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import { Command, Options } from "@effect/cli";
|
|
2
|
+
import { Chunk, Effect, Option, Ref, Stream } from "effect";
|
|
3
|
+
import { ParseResult } from "effect";
|
|
4
|
+
import type { Post } from "../domain/post.js";
|
|
5
|
+
import { FilterRuntime } from "../services/filter-runtime.js";
|
|
6
|
+
import { PostParser } from "../services/post-parser.js";
|
|
7
|
+
import { CliInput } from "./input.js";
|
|
8
|
+
import { CliInputError, CliJsonError } from "./errors.js";
|
|
9
|
+
import { parseFilterExpr } from "./filter-input.js";
|
|
10
|
+
import { decodeJson } from "./parse.js";
|
|
11
|
+
import { PipeInput, isRawPostInput, isStorePostInput } from "./pipe-input.js";
|
|
12
|
+
import { withExamples } from "./help.js";
|
|
13
|
+
import { filterOption, filterJsonOption } from "./shared-options.js";
|
|
14
|
+
import { formatSchemaError } from "./shared.js";
|
|
15
|
+
import { writeJsonStream } from "./output.js";
|
|
16
|
+
import { filterByFlags } from "../typeclass/chunk.js";
|
|
17
|
+
import { logErrorEvent, logWarn } from "./logging.js";
|
|
18
|
+
import { PositiveInt } from "./option-schemas.js";
|
|
19
|
+
|
|
20
|
+
const onErrorOption = Options.choice("on-error", ["fail", "skip", "report"]).pipe(
|
|
21
|
+
Options.withDescription("Behavior on invalid input lines"),
|
|
22
|
+
Options.withDefault("fail" as const)
|
|
23
|
+
);
|
|
24
|
+
|
|
25
|
+
const batchSizeOption = Options.integer("batch-size").pipe(
|
|
26
|
+
Options.withSchema(PositiveInt),
|
|
27
|
+
Options.withDescription("Posts per filter batch (default: 50)"),
|
|
28
|
+
Options.optional
|
|
29
|
+
);
|
|
30
|
+
|
|
31
|
+
const requireFilterExpr = (
|
|
32
|
+
filter: Option.Option<string>,
|
|
33
|
+
filterJson: Option.Option<string>
|
|
34
|
+
) =>
|
|
35
|
+
Option.isNone(filter) && Option.isNone(filterJson)
|
|
36
|
+
? Effect.fail(
|
|
37
|
+
CliInputError.make({
|
|
38
|
+
message: "Provide --filter or --filter-json.",
|
|
39
|
+
cause: { filter: null, filterJson: null }
|
|
40
|
+
})
|
|
41
|
+
)
|
|
42
|
+
: Effect.void;
|
|
43
|
+
|
|
44
|
+
const truncate = (value: string, max = 500) =>
|
|
45
|
+
value.length > max ? `${value.slice(0, max)}...` : value;
|
|
46
|
+
|
|
47
|
+
const formatPipeError = (error: unknown) => {
|
|
48
|
+
if (error instanceof CliJsonError || error instanceof CliInputError) {
|
|
49
|
+
return error.message;
|
|
50
|
+
}
|
|
51
|
+
if (ParseResult.isParseError(error)) {
|
|
52
|
+
return formatSchemaError(error);
|
|
53
|
+
}
|
|
54
|
+
if (typeof error === "object" && error !== null && "message" in error) {
|
|
55
|
+
const message = (error as { readonly message?: unknown }).message;
|
|
56
|
+
if (typeof message === "string" && message.length > 0) {
|
|
57
|
+
return message;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return String(error);
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
export const pipeCommand = Command.make(
|
|
64
|
+
"pipe",
|
|
65
|
+
{ filter: filterOption, filterJson: filterJsonOption, onError: onErrorOption, batchSize: batchSizeOption },
|
|
66
|
+
({ filter, filterJson, onError, batchSize }) =>
|
|
67
|
+
Effect.gen(function* () {
|
|
68
|
+
if (process.stdin.isTTY) {
|
|
69
|
+
return yield* CliInputError.make({
|
|
70
|
+
message: "stdin is a TTY. Pipe NDJSON input into skygent pipe.",
|
|
71
|
+
cause: { isTTY: true }
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
yield* requireFilterExpr(filter, filterJson);
|
|
75
|
+
|
|
76
|
+
const input = yield* CliInput;
|
|
77
|
+
const parser = yield* PostParser;
|
|
78
|
+
const runtime = yield* FilterRuntime;
|
|
79
|
+
const expr = yield* parseFilterExpr(filter, filterJson);
|
|
80
|
+
const evaluateBatch = yield* runtime.evaluateBatch(expr);
|
|
81
|
+
|
|
82
|
+
const size = Option.getOrElse(batchSize, () => 50);
|
|
83
|
+
|
|
84
|
+
const lineRef = yield* Ref.make(0);
|
|
85
|
+
const parsed = input.lines.pipe(
|
|
86
|
+
Stream.map((line) => line.trim()),
|
|
87
|
+
Stream.filter((line) => line.length > 0),
|
|
88
|
+
Stream.mapEffect((line) =>
|
|
89
|
+
Ref.updateAndGet(lineRef, (value) => value + 1).pipe(
|
|
90
|
+
Effect.map((lineNumber) => ({ line, lineNumber }))
|
|
91
|
+
)
|
|
92
|
+
),
|
|
93
|
+
Stream.mapEffect(({ line, lineNumber }) =>
|
|
94
|
+
decodeJson(PipeInput, line).pipe(
|
|
95
|
+
Effect.flatMap((inputPost) => {
|
|
96
|
+
if (isRawPostInput(inputPost)) {
|
|
97
|
+
return parser.parsePost(inputPost);
|
|
98
|
+
}
|
|
99
|
+
if (isStorePostInput(inputPost)) {
|
|
100
|
+
return Effect.succeed(inputPost.post);
|
|
101
|
+
}
|
|
102
|
+
return Effect.succeed(inputPost);
|
|
103
|
+
}),
|
|
104
|
+
Effect.map(Option.some),
|
|
105
|
+
Effect.catchAll((error) => {
|
|
106
|
+
if (onError === "fail") {
|
|
107
|
+
return Effect.fail(error);
|
|
108
|
+
}
|
|
109
|
+
const message = formatPipeError(error);
|
|
110
|
+
const payload = {
|
|
111
|
+
line: lineNumber,
|
|
112
|
+
message,
|
|
113
|
+
input: truncate(line)
|
|
114
|
+
};
|
|
115
|
+
const log =
|
|
116
|
+
onError === "report"
|
|
117
|
+
? logErrorEvent("Invalid input line", payload)
|
|
118
|
+
: logWarn("Skipping invalid input line", payload);
|
|
119
|
+
return log.pipe(
|
|
120
|
+
Effect.ignore,
|
|
121
|
+
Effect.as(Option.none<Post>())
|
|
122
|
+
);
|
|
123
|
+
})
|
|
124
|
+
)
|
|
125
|
+
),
|
|
126
|
+
Stream.filterMap((value) => value)
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
const filtered = parsed.pipe(
|
|
130
|
+
Stream.grouped(size),
|
|
131
|
+
Stream.mapEffect((batch) =>
|
|
132
|
+
evaluateBatch(batch).pipe(
|
|
133
|
+
Effect.map((flags) => filterByFlags(batch, flags))
|
|
134
|
+
)
|
|
135
|
+
),
|
|
136
|
+
Stream.mapConcat((chunk) => Chunk.toReadonlyArray(chunk))
|
|
137
|
+
);
|
|
138
|
+
|
|
139
|
+
yield* writeJsonStream(filtered);
|
|
140
|
+
})
|
|
141
|
+
).pipe(
|
|
142
|
+
Command.withDescription(
|
|
143
|
+
withExamples(
|
|
144
|
+
"Filter raw post NDJSON from stdin",
|
|
145
|
+
[
|
|
146
|
+
"skygent pipe --filter 'hashtag:#ai' < posts.ndjson",
|
|
147
|
+
"cat posts.ndjson | skygent pipe --filter-json '{\"_tag\":\"All\"}'"
|
|
148
|
+
],
|
|
149
|
+
[
|
|
150
|
+
"Note: stdin must be raw post NDJSON or skygent post NDJSON (from query --format ndjson)."
|
|
151
|
+
]
|
|
152
|
+
)
|
|
153
|
+
)
|
|
154
|
+
);
|
package/src/cli/post.ts
CHANGED
|
@@ -2,24 +2,29 @@ import { Command, Options } from "@effect/cli";
|
|
|
2
2
|
import { Context, Effect, Option, Stream } from "effect";
|
|
3
3
|
import { BskyClient } from "../services/bsky-client.js";
|
|
4
4
|
import { PostParser } from "../services/post-parser.js";
|
|
5
|
-
import type { PostLike
|
|
5
|
+
import type { PostLike } from "../domain/bsky.js";
|
|
6
6
|
import type { RawPost } from "../domain/raw.js";
|
|
7
7
|
import { renderPostsTable } from "../domain/format.js";
|
|
8
8
|
import { AppConfigService } from "../services/app-config.js";
|
|
9
9
|
import { withExamples } from "./help.js";
|
|
10
|
-
import { postUriArg
|
|
10
|
+
import { postUriArg } from "./shared-options.js";
|
|
11
|
+
import { PostCid } from "../domain/primitives.js";
|
|
11
12
|
import { writeJson, writeJsonStream, writeText } from "./output.js";
|
|
12
13
|
import { renderTableLegacy } from "./doc/table.js";
|
|
13
|
-
import {
|
|
14
|
+
import { renderProfileTable } from "./doc/table-renderers.js";
|
|
15
|
+
import { jsonNdjsonTableFormats } from "./output-format.js";
|
|
16
|
+
import { emitWithFormat } from "./output-render.js";
|
|
17
|
+
import { cursorOption as baseCursorOption, limitOption as baseLimitOption, parsePagination } from "./pagination.js";
|
|
18
|
+
import { CliInputError } from "./errors.js";
|
|
19
|
+
import { CliPreferences } from "./preferences.js";
|
|
20
|
+
import { compactPost, compactPostLike, compactProfileView } from "./compact-output.js";
|
|
14
21
|
|
|
15
|
-
const limitOption =
|
|
16
|
-
Options.withDescription("Maximum number of results")
|
|
17
|
-
Options.optional
|
|
22
|
+
const limitOption = baseLimitOption.pipe(
|
|
23
|
+
Options.withDescription("Maximum number of results")
|
|
18
24
|
);
|
|
19
25
|
|
|
20
|
-
const cursorOption =
|
|
21
|
-
Options.withDescription("Pagination cursor")
|
|
22
|
-
Options.optional
|
|
26
|
+
const cursorOption = baseCursorOption.pipe(
|
|
27
|
+
Options.withDescription("Pagination cursor")
|
|
23
28
|
);
|
|
24
29
|
|
|
25
30
|
const formatOption = Options.choice("format", jsonNdjsonTableFormats).pipe(
|
|
@@ -27,23 +32,23 @@ const formatOption = Options.choice("format", jsonNdjsonTableFormats).pipe(
|
|
|
27
32
|
Options.optional
|
|
28
33
|
);
|
|
29
34
|
|
|
35
|
+
const ensureSupportedFormat = (
|
|
36
|
+
format: Option.Option<typeof jsonNdjsonTableFormats[number]>,
|
|
37
|
+
configFormat: string
|
|
38
|
+
) =>
|
|
39
|
+
Option.isNone(format) && configFormat === "markdown"
|
|
40
|
+
? CliInputError.make({
|
|
41
|
+
message: 'Output format "markdown" is not supported for post commands. Use --format json|ndjson|table.',
|
|
42
|
+
cause: { format: configFormat }
|
|
43
|
+
})
|
|
44
|
+
: Effect.void;
|
|
45
|
+
|
|
30
46
|
const cidOption = Options.text("cid").pipe(
|
|
47
|
+
Options.withSchema(PostCid),
|
|
31
48
|
Options.withDescription("Filter engagement by specific record CID"),
|
|
32
49
|
Options.optional
|
|
33
50
|
);
|
|
34
51
|
|
|
35
|
-
const renderProfileTable = (
|
|
36
|
-
actors: ReadonlyArray<ProfileView>,
|
|
37
|
-
cursor: string | undefined
|
|
38
|
-
) => {
|
|
39
|
-
const rows = actors.map((actor) => [
|
|
40
|
-
actor.handle,
|
|
41
|
-
actor.displayName ?? "",
|
|
42
|
-
actor.did
|
|
43
|
-
]);
|
|
44
|
-
const table = renderTableLegacy(["HANDLE", "DISPLAY NAME", "DID"], rows);
|
|
45
|
-
return cursor ? `${table}\n\nCursor: ${cursor}` : table;
|
|
46
|
-
};
|
|
47
52
|
|
|
48
53
|
const renderLikesTable = (likes: ReadonlyArray<PostLike>, cursor: string | undefined) => {
|
|
49
54
|
const rows = likes.map((like) => [
|
|
@@ -71,28 +76,33 @@ const likesCommand = Command.make(
|
|
|
71
76
|
({ uri, cid, limit, cursor, format }) =>
|
|
72
77
|
Effect.gen(function* () {
|
|
73
78
|
const appConfig = yield* AppConfigService;
|
|
79
|
+
yield* ensureSupportedFormat(format, appConfig.outputFormat);
|
|
80
|
+
const preferences = yield* CliPreferences;
|
|
74
81
|
const client = yield* BskyClient;
|
|
75
|
-
const
|
|
82
|
+
const { limit: limitValue, cursor: cursorValue } = parsePagination(limit, cursor);
|
|
76
83
|
const result = yield* client.getLikes(uri, {
|
|
77
|
-
...(
|
|
78
|
-
...(
|
|
84
|
+
...(limitValue !== undefined ? { limit: limitValue } : {}),
|
|
85
|
+
...(cursorValue !== undefined ? { cursor: cursorValue } : {}),
|
|
79
86
|
...(Option.isSome(cid) ? { cid: cid.value } : {})
|
|
80
87
|
});
|
|
81
|
-
const
|
|
88
|
+
const likes = preferences.compact
|
|
89
|
+
? result.likes.map(compactPostLike)
|
|
90
|
+
: result.likes;
|
|
91
|
+
const payload = result.cursor ? { likes, cursor: result.cursor } : { likes };
|
|
92
|
+
const likesStream = Stream.fromIterable(
|
|
93
|
+
likes as ReadonlyArray<unknown>
|
|
94
|
+
);
|
|
95
|
+
yield* emitWithFormat(
|
|
82
96
|
format,
|
|
83
97
|
appConfig.outputFormat,
|
|
84
98
|
jsonNdjsonTableFormats,
|
|
85
|
-
"json"
|
|
99
|
+
"json",
|
|
100
|
+
{
|
|
101
|
+
json: writeJson(payload),
|
|
102
|
+
ndjson: writeJsonStream(likesStream),
|
|
103
|
+
table: writeText(renderLikesTable(result.likes, result.cursor))
|
|
104
|
+
}
|
|
86
105
|
);
|
|
87
|
-
if (outputFormat === "ndjson") {
|
|
88
|
-
yield* writeJsonStream(Stream.fromIterable(result.likes));
|
|
89
|
-
return;
|
|
90
|
-
}
|
|
91
|
-
if (outputFormat === "table") {
|
|
92
|
-
yield* writeText(renderLikesTable(result.likes, result.cursor));
|
|
93
|
-
return;
|
|
94
|
-
}
|
|
95
|
-
yield* writeJson(result);
|
|
96
106
|
})
|
|
97
107
|
).pipe(
|
|
98
108
|
Command.withDescription(
|
|
@@ -109,28 +119,35 @@ const repostedByCommand = Command.make(
|
|
|
109
119
|
({ uri, cid, limit, cursor, format }) =>
|
|
110
120
|
Effect.gen(function* () {
|
|
111
121
|
const appConfig = yield* AppConfigService;
|
|
122
|
+
yield* ensureSupportedFormat(format, appConfig.outputFormat);
|
|
123
|
+
const preferences = yield* CliPreferences;
|
|
112
124
|
const client = yield* BskyClient;
|
|
113
|
-
const
|
|
125
|
+
const { limit: limitValue, cursor: cursorValue } = parsePagination(limit, cursor);
|
|
114
126
|
const result = yield* client.getRepostedBy(uri, {
|
|
115
|
-
...(
|
|
116
|
-
...(
|
|
127
|
+
...(limitValue !== undefined ? { limit: limitValue } : {}),
|
|
128
|
+
...(cursorValue !== undefined ? { cursor: cursorValue } : {}),
|
|
117
129
|
...(Option.isSome(cid) ? { cid: cid.value } : {})
|
|
118
130
|
});
|
|
119
|
-
const
|
|
131
|
+
const repostedBy = preferences.compact
|
|
132
|
+
? result.repostedBy.map(compactProfileView)
|
|
133
|
+
: result.repostedBy;
|
|
134
|
+
const payload = result.cursor
|
|
135
|
+
? { repostedBy, cursor: result.cursor }
|
|
136
|
+
: { repostedBy };
|
|
137
|
+
const repostedStream = Stream.fromIterable(
|
|
138
|
+
repostedBy as ReadonlyArray<unknown>
|
|
139
|
+
);
|
|
140
|
+
yield* emitWithFormat(
|
|
120
141
|
format,
|
|
121
142
|
appConfig.outputFormat,
|
|
122
143
|
jsonNdjsonTableFormats,
|
|
123
|
-
"json"
|
|
144
|
+
"json",
|
|
145
|
+
{
|
|
146
|
+
json: writeJson(payload),
|
|
147
|
+
ndjson: writeJsonStream(repostedStream),
|
|
148
|
+
table: writeText(renderProfileTable(result.repostedBy, result.cursor))
|
|
149
|
+
}
|
|
124
150
|
);
|
|
125
|
-
if (outputFormat === "ndjson") {
|
|
126
|
-
yield* writeJsonStream(Stream.fromIterable(result.repostedBy));
|
|
127
|
-
return;
|
|
128
|
-
}
|
|
129
|
-
if (outputFormat === "table") {
|
|
130
|
-
yield* writeText(renderProfileTable(result.repostedBy, result.cursor));
|
|
131
|
-
return;
|
|
132
|
-
}
|
|
133
|
-
yield* writeJson(result);
|
|
134
151
|
})
|
|
135
152
|
).pipe(
|
|
136
153
|
Command.withDescription(
|
|
@@ -146,33 +163,38 @@ const quotesCommand = Command.make(
|
|
|
146
163
|
({ uri, cid, limit, cursor, format }) =>
|
|
147
164
|
Effect.gen(function* () {
|
|
148
165
|
const appConfig = yield* AppConfigService;
|
|
166
|
+
yield* ensureSupportedFormat(format, appConfig.outputFormat);
|
|
167
|
+
const preferences = yield* CliPreferences;
|
|
149
168
|
const client = yield* BskyClient;
|
|
150
169
|
const parser = yield* PostParser;
|
|
151
|
-
const
|
|
170
|
+
const { limit: limitValue, cursor: cursorValue } = parsePagination(limit, cursor);
|
|
152
171
|
const result = yield* client.getQuotes(uri, {
|
|
153
|
-
...(
|
|
154
|
-
...(
|
|
172
|
+
...(limitValue !== undefined ? { limit: limitValue } : {}),
|
|
173
|
+
...(cursorValue !== undefined ? { cursor: cursorValue } : {}),
|
|
155
174
|
...(Option.isSome(cid) ? { cid: cid.value } : {})
|
|
156
175
|
});
|
|
157
176
|
const posts = yield* parseRawPosts(parser, result.posts);
|
|
158
|
-
const
|
|
177
|
+
const compactPosts = preferences.compact
|
|
178
|
+
? posts.map(compactPost)
|
|
179
|
+
: posts;
|
|
180
|
+
const payload = {
|
|
181
|
+
...result,
|
|
182
|
+
posts: compactPosts
|
|
183
|
+
};
|
|
184
|
+
const postStream = Stream.fromIterable(
|
|
185
|
+
compactPosts as ReadonlyArray<unknown>
|
|
186
|
+
);
|
|
187
|
+
yield* emitWithFormat(
|
|
159
188
|
format,
|
|
160
189
|
appConfig.outputFormat,
|
|
161
190
|
jsonNdjsonTableFormats,
|
|
162
|
-
"json"
|
|
191
|
+
"json",
|
|
192
|
+
{
|
|
193
|
+
json: writeJson(payload),
|
|
194
|
+
ndjson: writeJsonStream(postStream),
|
|
195
|
+
table: writeText(renderPostsTable(posts))
|
|
196
|
+
}
|
|
163
197
|
);
|
|
164
|
-
if (outputFormat === "ndjson") {
|
|
165
|
-
yield* writeJsonStream(Stream.fromIterable(posts));
|
|
166
|
-
return;
|
|
167
|
-
}
|
|
168
|
-
if (outputFormat === "table") {
|
|
169
|
-
yield* writeText(renderPostsTable(posts));
|
|
170
|
-
return;
|
|
171
|
-
}
|
|
172
|
-
yield* writeJson({
|
|
173
|
-
...result,
|
|
174
|
-
posts
|
|
175
|
-
});
|
|
176
198
|
})
|
|
177
199
|
).pipe(
|
|
178
200
|
Command.withDescription(
|