@mepuka/skygent 0.2.0

package/src/domain/format.ts

@@ -0,0 +1,91 @@
+import { Post } from "./post.js";
+import { displayWidth, padEndDisplay } from "./text-width.js";
+
+const headers = ["Created At", "Author", "Text", "URI"];
+const textLimit = 80;
+
+export const normalizeWhitespace = (text: string) =>
+  text
+    .replace(/\r\n/g, "\n")
+    .replace(/\r/g, "\n")
+    .split("\n")
+    .map((line) => line.replace(/[ \t]+/g, " ").trim())
+    .join("\n")
+    .trim();
+
+export const collapseWhitespace = (text: string) =>
+  normalizeWhitespace(text).replace(/\n+/g, " ").trim();
+
+export const truncate = (text: string, max: number) => {
+  if (text.length <= max) return text;
+  if (max <= 3) return text.slice(0, max);
+  return `${text.slice(0, max - 3)}...`;
+};
+
+const sanitizeText = (text: string) => truncate(collapseWhitespace(text), textLimit);
+
+const sanitizeMarkdown = (text: string) =>
+  sanitizeText(text).replace(/[\\|*_`\[\]]/g, "\\$&");
+
+const postToRow = (post: Post) => [
+  post.createdAt.toISOString(),
+  post.author,
+  sanitizeText(post.text),
+  post.uri
+];
+
+const postToMarkdownRow = (post: Post) => [
+  post.createdAt.toISOString(),
+  post.author,
+  sanitizeMarkdown(post.text),
+  post.uri.replace(/\|/g, "\\|")
+];
+
+const renderTable = (
+  head: ReadonlyArray<string>,
+  rows: ReadonlyArray<ReadonlyArray<string>>
+) => {
+  const widths = head.map((value, index) =>
+    Math.max(
+      displayWidth(value),
+      ...rows.map((row) => displayWidth(row[index] ?? ""))
+    )
+  );
+
+  const formatRow = (row: ReadonlyArray<string>) =>
+    row
+      .map((cell, index) => padEndDisplay(cell ?? "", widths[index] ?? 0))
+      .join("  ");
+
+  const header = formatRow(head);
+  const separator = widths.map((width) => "-".repeat(width)).join("  ");
+  const body = rows.map(formatRow);
+
+  return [header, separator, ...body].join("\n");
+};
+
+const renderMarkdownHeader = (head: ReadonlyArray<string>) => {
+  const header = `| ${head.join(" | ")} |`;
+  const separator = `| ${head.map((label) => "-".repeat(Math.max(label.length, 3))).join(" | ")} |`;
+  return `${header}\n${separator}`;
+};
+
+export const renderPostsMarkdownHeader = () => renderMarkdownHeader(headers);
+
+export const renderPostMarkdownRow = (post: Post) =>
+  `| ${postToMarkdownRow(post).join(" | ")} |`;
+
+const renderMarkdownTable = (
+  head: ReadonlyArray<string>,
+  rows: ReadonlyArray<ReadonlyArray<string>>
+) => {
+  const header = renderMarkdownHeader(head);
+  const body = rows.map((row) => `| ${row.join(" | ")} |`);
+  return [header, ...body].join("\n");
+};
+
+export const renderPostsTable = (posts: ReadonlyArray<Post>) =>
+  renderTable(headers, posts.map(postToRow));
+
+export const renderPostsMarkdown = (posts: ReadonlyArray<Post>) =>
+  renderMarkdownTable(headers, posts.map(postToMarkdownRow));

package/src/domain/index.ts

@@ -0,0 +1,13 @@
+export * from "./primitives.js";
+export * from "./post.js";
+export * from "./policies.js";
+export * from "./filter.js";
+export * from "./errors.js";
+export * from "./events.js";
+export * from "./store.js";
+export * from "./extract.js";
+export * from "./raw.js";
+export * from "./sync.js";
+export * from "./defaults.js";
+export * from "./bsky.js";
+export * from "./credentials.js";

package/src/domain/indexes.ts

@@ -0,0 +1,17 @@
+import { Schema } from "effect";
+import { EventSeq, Handle, Hashtag, PostUri, Timestamp } from "./primitives.js";
+
+export class PostIndexEntry extends Schema.Class<PostIndexEntry>("PostIndexEntry")({
+  uri: PostUri,
+  createdDate: Schema.String,
+  hashtags: Schema.Array(Hashtag),
+  author: Schema.optional(Handle)
+}) {}
+
+export class IndexCheckpoint extends Schema.Class<IndexCheckpoint>("IndexCheckpoint")({
+  index: Schema.String,
+  version: Schema.NonNegativeInt,
+  lastEventSeq: EventSeq,
+  eventCount: Schema.NonNegativeInt,
+  updatedAt: Timestamp
+}) {}

package/src/domain/policies.ts

@@ -0,0 +1,16 @@
+import { Schema } from "effect";
+
+export class IncludeOnError extends Schema.TaggedClass<IncludeOnError>()("Include", {}) {}
+export class ExcludeOnError extends Schema.TaggedClass<ExcludeOnError>()("Exclude", {}) {}
+
+export class RetryOnError extends Schema.TaggedClass<RetryOnError>()("Retry", {
+  maxRetries: Schema.NonNegativeInt,
+  baseDelay: Schema.Duration
+}) {}
+
+export const FilterErrorPolicy = Schema.Union(
+  IncludeOnError,
+  ExcludeOnError,
+  RetryOnError
+);
+export type FilterErrorPolicy = typeof FilterErrorPolicy.Type;

package/src/domain/post.ts

@@ -0,0 +1,88 @@
+import { Schema } from "effect";
+import { Did, Handle, Hashtag, PostCid, PostUri, Timestamp } from "./primitives.js";
+import {
+  FeedContext,
+  Label,
+  PostEmbed,
+  PostMetrics,
+  PostViewerState,
+  ProfileBasic,
+  ReplyRef,
+  RichTextFacet,
+  SelfLabel,
+  ThreadgateView
+} from "./bsky.js";
+
+/**
+ * A normalized Bluesky post suitable for storage and filtering.
+ *
+ * This is the core data model for Skygent. Posts are parsed from Bluesky's
+ * AT Protocol format and normalized to a consistent structure for querying
+ * and filtering. Each post captures the content, metadata, relationships,
+ * and engagement metrics from the original Bluesky post.
+ *
+ * @example
+ * ```ts
+ * const post = new Post({
+ *   uri: "at://did:plc:abc/app.bsky.feed.post/123",
+ *   author: "@alice.bsky.social",
+ *   text: "Hello, Bluesky! #introduction",
+ *   createdAt: "2024-01-15T10:30:00Z",
+ *   hashtags: ["introduction"],
+ *   mentions: [],
+ *   links: []
+ * });
+ * ```
+ */
+export class Post extends Schema.Class<Post>("Post")({
+  /** The AT Protocol URI uniquely identifying this post */
+  uri: PostUri,
+  /** The content identifier (CID) of the post record (optional for some contexts) */
+  cid: Schema.optional(PostCid),
+  /** The author's Bluesky handle (e.g., "@alice.bsky.social") */
+  author: Handle,
+  /** The author's decentralized identifier (DID) if resolved */
+  authorDid: Schema.optional(Did),
+  /** The author's profile information including display name and avatar */
+  authorProfile: Schema.optional(ProfileBasic),
+  /** The full text content of the post */
+  text: Schema.String,
+  /** ISO timestamp when the post was created */
+  createdAt: Timestamp,
+  /** Array of hashtags extracted from the post text (without # prefix) */
+  hashtags: Schema.Array(Hashtag),
+  /** Array of @mentions in the post (without @ prefix) */
+  mentions: Schema.Array(Handle),
+  /** Array of DIDs corresponding to the mentions if resolved */
+  mentionDids: Schema.optional(Schema.Array(Did)),
+  /** Array of external URLs extracted from the post */
+  links: Schema.Array(Schema.URL),
+  /** Rich text facets defining formatting and entity positions */
+  facets: Schema.optional(Schema.Array(RichTextFacet)),
+  /** Reference to the parent post if this is a reply */
+  reply: Schema.optional(ReplyRef),
+  /** Embedded media or records (images, videos, external links, quotes) */
+  embed: Schema.optional(PostEmbed),
+  /** The raw embedded record data for complex embed types */
+  recordEmbed: Schema.optional(Schema.Unknown),
+  /** Array of language codes for the post content */
+  langs: Schema.optional(Schema.Array(Schema.String)),
+  /** Array of user-defined tags on the post */
+  tags: Schema.optional(Schema.Array(Schema.String)),
+  /** Self-applied content labels for moderation */
+  selfLabels: Schema.optional(Schema.Array(SelfLabel)),
+  /** Moderation labels applied to this post */
+  labels: Schema.optional(Schema.Array(Label)),
+  /** Engagement metrics (likes, reposts, replies, quotes) */
+  metrics: Schema.optional(PostMetrics),
+  /** ISO timestamp when the post was indexed by Bluesky */
+  indexedAt: Schema.optional(Timestamp),
+  /** Viewer-specific state (like/repost status, thread muting) */
+  viewer: Schema.optional(PostViewerState),
+  /** Thread moderation settings if applied */
+  threadgate: Schema.optional(ThreadgateView),
+  /** Debug information for development purposes */
+  debug: Schema.optional(Schema.Unknown),
+  /** Context about how this post was retrieved (timeline, feed, etc.) */
+  feed: Schema.optional(FeedContext)
+}) {}

package/src/domain/primitives.ts

@@ -0,0 +1,50 @@
+import { Schema } from "effect";
+
+export const Handle = Schema.Lowercase.pipe(
+  Schema.pattern(/^[a-z0-9][a-z0-9.-]{1,251}$/),
+  Schema.brand("Handle")
+);
+export type Handle = typeof Handle.Type;
+
+// Bluesky's lexicon only requires: maxLength 640 bytes, maxGraphemes 64
+// The extraction regex is stricter, but facets can contain anything
+// We accept whatever Bluesky sends to avoid losing posts
+// Require at least one non-whitespace character after #
+export const Hashtag = Schema.String.pipe(
+  Schema.pattern(/^#\S.*$/u),
+  Schema.brand("Hashtag")
+);
+export type Hashtag = typeof Hashtag.Type;
+
+export const AtUri = Schema.String.pipe(Schema.brand("AtUri"));
+export type AtUri = typeof AtUri.Type;
+
+export const PostUri = Schema.String.pipe(Schema.brand("PostUri"));
+export type PostUri = typeof PostUri.Type;
+
+export const PostCid = Schema.String.pipe(Schema.brand("PostCid"));
+export type PostCid = typeof PostCid.Type;
+
+export const Did = Schema.String.pipe(Schema.brand("Did"));
+export type Did = typeof Did.Type;
+
+export const Timestamp = Schema.Union(
+  Schema.DateFromString,
+  Schema.DateFromSelf
+).pipe(Schema.brand("Timestamp"));
+export type Timestamp = typeof Timestamp.Type;
+
+export const EventId = Schema.ULID.pipe(Schema.brand("EventId"));
+export type EventId = typeof EventId.Type;
+
+export const EventSeq = Schema.NonNegativeInt.pipe(Schema.brand("EventSeq"));
+export type EventSeq = typeof EventSeq.Type;
+
+export const StoreName = Schema.String.pipe(
+  Schema.pattern(/^[a-z0-9][a-z0-9-_]{1,63}$/),
+  Schema.brand("StoreName")
+);
+export type StoreName = typeof StoreName.Type;
+
+export const StorePath = Schema.String.pipe(Schema.brand("StorePath"));
+export type StorePath = typeof StorePath.Type;

package/src/domain/raw.ts

@@ -0,0 +1,140 @@
+import { Effect, ParseResult, Schema } from "effect";
+import { extractFromFacets, extractHashtags, extractLinks, extractMentions } from "./extract.js";
+import { Post } from "./post.js";
+import { Did, Handle, PostCid, PostUri } from "./primitives.js";
+import {
+  EmbedUnknown,
+  FeedContext,
+  Label,
+  LegacyEntity,
+  PostEmbed,
+  PostMetrics,
+  PostViewerState,
+  ProfileBasic,
+  ReplyRef,
+  RichTextFacet,
+  SelfLabels,
+  ThreadgateView
+} from "./bsky.js";
+
+export class RawPostRecord extends Schema.Class<RawPostRecord>("RawPostRecord")({
+  $type: Schema.optional(Schema.Literal("app.bsky.feed.post")),
+  text: Schema.String,
+  createdAt: Schema.String,
+  facets: Schema.optional(Schema.Array(RichTextFacet)),
+  entities: Schema.optional(Schema.Array(LegacyEntity)),
+  reply: Schema.optional(ReplyRef),
+  embed: Schema.optional(Schema.Unknown),
+  langs: Schema.optional(Schema.Array(Schema.String)),
+  labels: Schema.optional(SelfLabels),
+  tags: Schema.optional(Schema.Array(Schema.String))
+}) {}
+
+export class RawPost extends Schema.Class<RawPost>("RawPost")({
+  uri: PostUri,
+  cid: Schema.optional(PostCid),
+  author: Handle,
+  authorDid: Schema.optional(Did),
+  authorProfile: Schema.optional(ProfileBasic),
+  record: RawPostRecord,
+  indexedAt: Schema.optional(Schema.String),
+  labels: Schema.optional(Schema.Array(Schema.encodedSchema(Label))),
+  metrics: Schema.optional(PostMetrics),
+  embed: Schema.optional(PostEmbed),
+  viewer: Schema.optional(PostViewerState),
+  threadgate: Schema.optional(ThreadgateView),
+  debug: Schema.optional(Schema.Unknown),
+  feed: Schema.optional(FeedContext),
+  _pageCursor: Schema.optional(Schema.String)
+}) {}
+
+export const PostFromRaw = Schema.transformOrFail(RawPost, Post, {
+  strict: true,
+  decode: (raw) => {
+    const unique = <T>(items: ReadonlyArray<T>) => Array.from(new Set(items));
+    const facetData = extractFromFacets(raw.record.facets);
+    const tagOverrides =
+      raw.record.tags?.map((tag) =>
+        tag.startsWith("#") ? tag : `#${tag}`
+      ) ?? [];
+    const hashtags = unique([
+      ...extractHashtags(raw.record.text),
+      ...facetData.hashtags,
+      ...tagOverrides
+    ]);
+    const mentions = unique(extractMentions(raw.record.text));
+    const links = unique([
+      ...extractLinks(raw.record.text),
+      ...facetData.links
+    ]);
+    const embed =
+      raw.embed ??
+      (raw.record.embed
+        ? EmbedUnknown.make({
+            rawType:
+              typeof (raw.record.embed as { $type?: unknown })?.$type === "string"
+                ? String((raw.record.embed as { $type?: unknown }).$type)
+                : "unknown",
+            data: raw.record.embed
+          })
+        : undefined);
+    return ParseResult.decodeUnknown(Schema.encodedSchema(Post))({
+      uri: raw.uri,
+      cid: raw.cid,
+      author: raw.author,
+      authorDid: raw.authorDid,
+      authorProfile: raw.authorProfile,
+      text: raw.record.text,
+      createdAt: raw.record.createdAt,
+      hashtags,
+      mentions,
+      mentionDids: facetData.mentionDids,
+      links,
+      facets: raw.record.facets,
+      reply: raw.record.reply,
+      embed,
+      langs: raw.record.langs,
+      tags: raw.record.tags,
+      selfLabels: raw.record.labels?.values,
+      labels: raw.labels,
+      metrics: raw.metrics,
+      indexedAt: raw.indexedAt,
+      viewer: raw.viewer,
+      threadgate: raw.threadgate,
+      debug: raw.debug,
+      feed: raw.feed,
+      recordEmbed: raw.record.embed
+    });
+  },
+  encode: (_encoded, _options, _ast, post) =>
+    Effect.gen(function* () {
+      const labels = post.labels
+        ? yield* ParseResult.encodeUnknown(Schema.Array(Label))(post.labels)
+        : undefined;
+      return yield* ParseResult.decodeUnknown(RawPost)({
+        uri: post.uri,
+        cid: post.cid,
+        author: post.author,
+        authorDid: post.authorDid,
+        authorProfile: post.authorProfile,
+        indexedAt: post.indexedAt?.toISOString(),
+        labels,
+        metrics: post.metrics,
+        embed: post.embed,
+        viewer: post.viewer,
+        threadgate: post.threadgate,
+        debug: post.debug,
+        feed: post.feed,
+        record: {
+          text: post.text,
+          createdAt: post.createdAt.toISOString(),
+          facets: post.facets,
+          reply: post.reply,
+          embed: post.recordEmbed,
+          langs: post.langs,
+          labels: post.selfLabels ? { values: post.selfLabels } : undefined,
+          tags: post.tags
+        }
+      });
+    })
+});

package/src/domain/store.ts

@@ -0,0 +1,103 @@
+import { Schema } from "effect";
+import { FilterExprSchema } from "./filter.js";
+import { StoreName, StorePath, Timestamp } from "./primitives.js";
+
+/**
+ * A reference to a Skygent store, uniquely identified by its name and root directory.
+ *
+ * Stores are the primary data containers in Skygent, holding filtered Bluesky posts.
+ * Each store maintains its own SQLite database and event log.
+ *
+ * @example
+ * ```ts
+ * const storeRef = new StoreRef({
+ *   name: "tech-posts",
+ *   root: "/path/to/.skygent"
+ * });
+ * ```
+ */
+export class StoreRef extends Schema.Class<StoreRef>("StoreRef")({
+  /** The unique name of the store (2-64 characters, lowercase alphanumeric with hyphens and underscores) */
+  name: StoreName,
+  /** The absolute path to the store's root directory where database and files are stored */
+  root: StorePath
+}) {}
+
+/**
+ * Metadata about a store, tracking its creation and last update times.
+ *
+ * This is stored in the store's metadata file and used for tracking store state.
+ */
+export class StoreMetadata extends Schema.Class<StoreMetadata>("StoreMetadata")({
+  /** The unique name of the store */
+  name: StoreName,
+  /** The absolute path to the store's root directory */
+  root: StorePath,
+  /** ISO timestamp when the store was first created */
+  createdAt: Timestamp,
+  /** ISO timestamp when the store was last modified */
+  updatedAt: Timestamp
+}) {}
+
+/**
+ * Configuration for filter output format and destination.
+ *
+ * Defines how filtered posts should be exported from a store, including
+ * the output path and which formats (JSON, Markdown) to generate.
+ */
+export class FilterOutput extends Schema.Class<FilterOutput>("FilterOutput")({
+  /** The file system path where filtered output should be written */
+  path: Schema.String,
+  /** Whether to output posts as JSON */
+  json: Schema.Boolean,
+  /** Whether to output posts as Markdown */
+  markdown: Schema.Boolean
+}) {}
+
+/**
+ * A filter specification that defines which posts to include in a store.
+ *
+ * Each filter has a unique name within the store and an expression that
+ * determines which posts match. Filtered output can be exported to files.
+ */
+export class FilterSpec extends Schema.Class<FilterSpec>("FilterSpec")({
+  /** The unique name for this filter within the store */
+  name: Schema.String,
+  /** The filter expression that determines which posts to include */
+  expr: FilterExprSchema,
+  /** Configuration for how to output filtered posts */
+  output: FilterOutput
+}) {}
+
+/**
+ * Policy for handling duplicate posts during sync operations.
+ *
+ * - `dedupe`: Skip posts that already exist in the store (default)
+ * - `refresh`: Overwrite existing posts with fresh data from the API
+ */
+export const SyncUpsertPolicy = Schema.Literal("dedupe", "refresh");
+export type SyncUpsertPolicy = typeof SyncUpsertPolicy.Type;
+
+/**
+ * Complete configuration for a store.
+ *
+ * Defines the store's behavior including default output formats, automatic
+ * sync settings, and the set of filters that populate the store.
+ */
+export class StoreConfig extends Schema.Class<StoreConfig>("StoreConfig")({
+  /** Default output format settings for the store */
+  format: Schema.Struct({
+    /** Whether to enable JSON output by default */
+    json: Schema.Boolean,
+    /** Whether to enable Markdown output by default */
+    markdown: Schema.Boolean
+  }),
+  /** Whether to automatically sync when running watch mode */
+  autoSync: Schema.Boolean,
+  /** Optional ISO 8601 duration string for automatic sync intervals (e.g., "PT5M") */
+  syncInterval: Schema.optional(Schema.String),
+  /** Policy for handling duplicate posts during sync (defaults to "dedupe") */
+  syncPolicy: Schema.optional(SyncUpsertPolicy),
+  /** Array of filter specifications that determine which posts to store */
+  filters: Schema.Array(FilterSpec)
+}) {}