@gokulvenkatareddy/cortex 0.1.7

package/apps/octogent/apps/api/src/monitor/repository.ts

@@ -0,0 +1,303 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+import {
+  DEFAULT_MONITOR_MAX_CACHE_AGE_MS,
+  DEFAULT_MONITOR_MAX_POSTS,
+  DEFAULT_MONITOR_SEARCH_WINDOW_DAYS,
+} from "./defaults";
+import type { MonitorRepository, PersistedMonitorCache, PersistedMonitorConfig } from "./types";
+
+const MONITOR_CONFIG_VERSION = 1 as const;
+const MONITOR_CACHE_VERSION = 1 as const;
+const MONITOR_CONFIG_RELATIVE_PATH = ".octogent/state/monitor-config.json";
+const MONITOR_CACHE_RELATIVE_PATH = ".octogent/state/monitor-cache.json";
+const VALID_MONITOR_SEARCH_WINDOW_DAYS = new Set([1, 3, 7]);
+
+const asRecord = (value: unknown): Record<string, unknown> | null =>
+  value !== null && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+
+const readJsonDocument = (path: string): unknown => {
+  const raw = readFileSync(path, "utf8");
+  return JSON.parse(raw);
+};
+
+const normalizeStringArray = (value: unknown): string[] => {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+
+  return value
+    .filter((item): item is string => typeof item === "string")
+    .map((item) => item.trim())
+    .filter((item) => item.length > 0);
+};
+
+const normalizeMonitorPost = (value: unknown): PersistedMonitorCache["posts"][number] | null => {
+  const record = asRecord(value);
+  if (!record) {
+    return null;
+  }
+
+  const source = record.source;
+  const id = typeof record.id === "string" ? record.id : null;
+  const text = typeof record.text === "string" ? record.text : null;
+  const author = typeof record.author === "string" ? record.author : null;
+  const createdAt = typeof record.createdAt === "string" ? record.createdAt : null;
+  const likeCount =
+    typeof record.likeCount === "number" && Number.isFinite(record.likeCount)
+      ? Math.max(0, Math.floor(record.likeCount))
+      : null;
+  const permalink = typeof record.permalink === "string" ? record.permalink : null;
+  const matchedQueryTerm =
+    typeof record.matchedQueryTerm === "string" ? record.matchedQueryTerm : null;
+
+  if (
+    source !== "x" ||
+    id === null ||
+    text === null ||
+    author === null ||
+    createdAt === null ||
+    likeCount === null ||
+    permalink === null
+  ) {
+    return null;
+  }
+
+  return {
+    source,
+    id,
+    text,
+    author,
+    createdAt,
+    likeCount,
+    permalink,
+    matchedQueryTerm,
+  };
+};
+
+const normalizeUsage = (value: unknown): PersistedMonitorCache["usage"] => {
+  const record = asRecord(value);
+  if (!record) {
+    return null;
+  }
+
+  const status = record.status;
+  if (status !== "ok" && status !== "unavailable" && status !== "error") {
+    return null;
+  }
+
+  const source = record.source === "x-api" ? "x-api" : "none";
+  const fetchedAt =
+    typeof record.fetchedAt === "string" ? record.fetchedAt : new Date().toISOString();
+  const message = typeof record.message === "string" ? record.message : null;
+
+  const cap =
+    typeof record.cap === "number" && Number.isFinite(record.cap) ? Math.max(0, record.cap) : null;
+  const used =
+    typeof record.used === "number" && Number.isFinite(record.used)
+      ? Math.max(0, record.used)
+      : null;
+  const remaining =
+    typeof record.remaining === "number" && Number.isFinite(record.remaining)
+      ? Math.max(0, record.remaining)
+      : null;
+  const resetAt = typeof record.resetAt === "string" ? record.resetAt : null;
+
+  return {
+    status,
+    source,
+    fetchedAt,
+    message,
+    cap,
+    used,
+    remaining,
+    resetAt,
+  };
+};
+
+const normalizeCredentials = (
+  value: unknown,
+): PersistedMonitorConfig["providers"]["x"]["credentials"] => {
+  const record = asRecord(value);
+  if (!record) {
+    return null;
+  }
+
+  const bearerToken = typeof record.bearerToken === "string" ? record.bearerToken.trim() : "";
+  if (bearerToken.length === 0) {
+    return null;
+  }
+
+  const normalizeOptional = (input: unknown) => {
+    if (typeof input !== "string") {
+      return null;
+    }
+    const trimmed = input.trim();
+    return trimmed.length > 0 ? trimmed : null;
+  };
+
+  return {
+    bearerToken,
+    apiKey: normalizeOptional(record.apiKey),
+    apiSecret: normalizeOptional(record.apiSecret),
+    accessToken: normalizeOptional(record.accessToken),
+    accessTokenSecret: normalizeOptional(record.accessTokenSecret),
+    updatedAt:
+      typeof record.updatedAt === "string" && record.updatedAt.length > 0
+        ? record.updatedAt
+        : new Date().toISOString(),
+  };
+};
+
+const defaultConfig = (): PersistedMonitorConfig => ({
+  version: MONITOR_CONFIG_VERSION,
+  providerId: "x",
+  queryTerms: [],
+  refreshPolicy: {
+    maxCacheAgeMs: DEFAULT_MONITOR_MAX_CACHE_AGE_MS,
+    maxPosts: DEFAULT_MONITOR_MAX_POSTS,
+    searchWindowDays: DEFAULT_MONITOR_SEARCH_WINDOW_DAYS,
+  },
+  providers: {
+    x: {
+      credentials: null,
+    },
+  },
+});
+
+const defaultCache = (): PersistedMonitorCache => ({
+  version: MONITOR_CACHE_VERSION,
+  providerId: "x",
+  queryTerms: defaultConfig().queryTerms,
+  fetchedAt: null,
+  lastError: null,
+  posts: [],
+  usage: null,
+});
+
+const normalizeConfig = (value: unknown): PersistedMonitorConfig => {
+  const fallback = defaultConfig();
+  const record = asRecord(value);
+  if (!record || record.version !== MONITOR_CONFIG_VERSION) {
+    return fallback;
+  }
+
+  const queryTerms = normalizeStringArray(record.queryTerms);
+  const refreshPolicyRecord = asRecord(record.refreshPolicy);
+  const maxCacheAgeMs =
+    refreshPolicyRecord &&
+    typeof refreshPolicyRecord.maxCacheAgeMs === "number" &&
+    Number.isFinite(refreshPolicyRecord.maxCacheAgeMs) &&
+    refreshPolicyRecord.maxCacheAgeMs > 0
+      ? Math.floor(refreshPolicyRecord.maxCacheAgeMs)
+      : fallback.refreshPolicy.maxCacheAgeMs;
+  const maxPosts =
+    refreshPolicyRecord &&
+    typeof refreshPolicyRecord.maxPosts === "number" &&
+    Number.isFinite(refreshPolicyRecord.maxPosts) &&
+    refreshPolicyRecord.maxPosts > 0
+      ? Math.floor(refreshPolicyRecord.maxPosts)
+      : fallback.refreshPolicy.maxPosts;
+  const searchWindowDaysRaw =
+    refreshPolicyRecord &&
+    typeof refreshPolicyRecord.searchWindowDays === "number" &&
+    Number.isFinite(refreshPolicyRecord.searchWindowDays)
+      ? Math.floor(refreshPolicyRecord.searchWindowDays)
+      : null;
+  const searchWindowDays =
+    searchWindowDaysRaw !== null && VALID_MONITOR_SEARCH_WINDOW_DAYS.has(searchWindowDaysRaw)
+      ? (searchWindowDaysRaw as 1 | 3 | 7)
+      : fallback.refreshPolicy.searchWindowDays;
+
+  const providersRecord = asRecord(record.providers);
+  const xProviderRecord = providersRecord ? asRecord(providersRecord.x) : null;
+
+  return {
+    version: MONITOR_CONFIG_VERSION,
+    providerId: record.providerId === "x" ? "x" : fallback.providerId,
+    queryTerms,
+    refreshPolicy: {
+      maxCacheAgeMs,
+      maxPosts,
+      searchWindowDays,
+    },
+    providers: {
+      x: {
+        credentials: normalizeCredentials(xProviderRecord?.credentials),
+      },
+    },
+  };
+};
+
+const normalizeCache = (value: unknown): PersistedMonitorCache => {
+  const fallback = defaultCache();
+  const record = asRecord(value);
+  if (!record || record.version !== MONITOR_CACHE_VERSION) {
+    return fallback;
+  }
+
+  const posts = Array.isArray(record.posts)
+    ? record.posts
+        .map((post) => normalizeMonitorPost(post))
+        .filter((post): post is NonNullable<typeof post> => post !== null)
+    : [];
+
+  const queryTerms = normalizeStringArray(record.queryTerms);
+
+  return {
+    version: MONITOR_CACHE_VERSION,
+    providerId: record.providerId === "x" ? "x" : fallback.providerId,
+    queryTerms,
+    fetchedAt: typeof record.fetchedAt === "string" ? record.fetchedAt : null,
+    lastError: typeof record.lastError === "string" ? record.lastError : null,
+    posts,
+    usage: normalizeUsage(record.usage),
+  };
+};
+
+const writeJsonDocument = (path: string, value: unknown) => {
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`, "utf8");
+};
+
+export const createFileMonitorRepository = (projectStateDir: string): MonitorRepository => {
+  const configPath = join(projectStateDir, "state", "monitor-config.json");
+  const cachePath = join(projectStateDir, "state", "monitor-cache.json");
+
+  return {
+    readConfig() {
+      if (!existsSync(configPath)) {
+        return defaultConfig();
+      }
+
+      try {
+        return normalizeConfig(readJsonDocument(configPath));
+      } catch {
+        return defaultConfig();
+      }
+    },
+
+    writeConfig(config) {
+      writeJsonDocument(configPath, config);
+    },
+
+    readCache() {
+      if (!existsSync(cachePath)) {
+        return defaultCache();
+      }
+
+      try {
+        return normalizeCache(readJsonDocument(cachePath));
+      } catch {
+        return defaultCache();
+      }
+    },
+
+    writeCache(cache) {
+      writeJsonDocument(cachePath, cache);
+    },
+  };
+};

package/apps/octogent/apps/api/src/monitor/service.ts

@@ -0,0 +1,349 @@
+import { createFileMonitorRepository } from "./repository";
+import type {
+  MonitorConfigPatchInput,
+  MonitorFeedSnapshot,
+  MonitorPost,
+  MonitorProviderAdapter,
+  MonitorProviderId,
+  MonitorReadFeedOptions,
+  MonitorRepository,
+  MonitorService,
+  PersistedMonitorCache,
+  PersistedMonitorConfig,
+  SanitizedMonitorConfig,
+} from "./types";
+import { createXMonitorProvider } from "./xProvider";
+
+export class MonitorInputError extends Error {}
+
+const normalizeQueryTerms = (queryTerms: string[]): string[] => {
+  const normalized = queryTerms.map((term) => term.trim()).filter((term) => term.length > 0);
+  return [...new Set(normalized)];
+};
+
+const normalizeDateMs = (value: string | null): number | null => {
+  if (!value) {
+    return null;
+  }
+
+  const ms = Date.parse(value);
+  return Number.isFinite(ms) ? ms : null;
+};
+
+const queryTermsMatch = (left: string[], right: string[]): boolean => {
+  if (left.length !== right.length) {
+    return false;
+  }
+
+  return left.every((term, index) => term === right[index]);
+};
+
+export const isMonitorCacheStale = ({
+  now,
+  maxCacheAgeMs,
+  lastFetchedAt,
+  cachedQueryTerms,
+  currentQueryTerms,
+}: {
+  now: Date;
+  maxCacheAgeMs: number;
+  lastFetchedAt: string | null;
+  cachedQueryTerms: string[];
+  currentQueryTerms: string[];
+}): boolean => {
+  if (!queryTermsMatch(cachedQueryTerms, currentQueryTerms)) {
+    return true;
+  }
+
+  const fetchedAtMs = normalizeDateMs(lastFetchedAt);
+  if (fetchedAtMs === null) {
+    return true;
+  }
+
+  return now.getTime() - fetchedAtMs >= maxCacheAgeMs;
+};
+
+const toRankDedupeKey = (post: MonitorPost) => `${post.source}:${post.id}`;
+
+export const rankAndLimitPostsByLikes = (posts: MonitorPost[], limit = 30): MonitorPost[] => {
+  const dedupedById = new Map<string, MonitorPost>();
+
+  for (const post of posts) {
+    const key = toRankDedupeKey(post);
+    const existing = dedupedById.get(key);
+    if (!existing) {
+      dedupedById.set(key, post);
+      continue;
+    }
+
+    if (post.likeCount > existing.likeCount) {
+      dedupedById.set(key, post);
+      continue;
+    }
+
+    if (post.likeCount === existing.likeCount) {
+      const postCreatedAtMs = normalizeDateMs(post.createdAt) ?? 0;
+      const existingCreatedAtMs = normalizeDateMs(existing.createdAt) ?? 0;
+      if (postCreatedAtMs > existingCreatedAtMs) {
+        dedupedById.set(key, post);
+      }
+    }
+  }
+
+  return [...dedupedById.values()]
+    .sort((left, right) => {
+      if (left.likeCount !== right.likeCount) {
+        return right.likeCount - left.likeCount;
+      }
+
+      const leftMs = normalizeDateMs(left.createdAt) ?? 0;
+      const rightMs = normalizeDateMs(right.createdAt) ?? 0;
+      return rightMs - leftMs;
+    })
+    .slice(0, Math.max(1, limit));
+};
+
+const buildStaleAfter = (lastFetchedAt: string | null, maxCacheAgeMs: number): string | null => {
+  const fetchedAtMs = normalizeDateMs(lastFetchedAt);
+  if (fetchedAtMs === null) {
+    return null;
+  }
+
+  return new Date(fetchedAtMs + maxCacheAgeMs).toISOString();
+};
+
+const sanitizeConfig = (
+  config: PersistedMonitorConfig,
+  providers: Map<MonitorProviderId, MonitorProviderAdapter>,
+): SanitizedMonitorConfig => {
+  const provider = providers.get(config.providerId);
+  if (!provider) {
+    throw new MonitorInputError(`Unsupported monitor provider: ${config.providerId}`);
+  }
+
+  return {
+    providerId: config.providerId,
+    queryTerms: [...config.queryTerms],
+    refreshPolicy: {
+      maxCacheAgeMs: config.refreshPolicy.maxCacheAgeMs,
+      maxPosts: config.refreshPolicy.maxPosts,
+      searchWindowDays: config.refreshPolicy.searchWindowDays,
+    },
+    providers: {
+      x: {
+        credentials: provider.summarizeCredentials(config.providers.x.credentials),
+      },
+    },
+  };
+};
+
+const toFeedSnapshot = (
+  config: PersistedMonitorConfig,
+  cache: PersistedMonitorCache,
+  now: Date,
+): MonitorFeedSnapshot => {
+  const isStale = isMonitorCacheStale({
+    now,
+    maxCacheAgeMs: config.refreshPolicy.maxCacheAgeMs,
+    lastFetchedAt: cache.fetchedAt,
+    cachedQueryTerms: cache.queryTerms,
+    currentQueryTerms: config.queryTerms,
+  });
+
+  return {
+    providerId: config.providerId,
+    queryTerms: [...config.queryTerms],
+    refreshPolicy: {
+      maxCacheAgeMs: config.refreshPolicy.maxCacheAgeMs,
+      maxPosts: config.refreshPolicy.maxPosts,
+      searchWindowDays: config.refreshPolicy.searchWindowDays,
+    },
+    lastFetchedAt: cache.fetchedAt,
+    staleAfter: buildStaleAfter(cache.fetchedAt, config.refreshPolicy.maxCacheAgeMs),
+    isStale,
+    lastError: cache.lastError,
+    posts: [...cache.posts],
+    usage: cache.usage,
+  };
+};
+
+export const createMonitorService = ({
+  projectStateDir,
+  repository = createFileMonitorRepository(projectStateDir),
+  providers = [createXMonitorProvider()],
+}: {
+  projectStateDir: string;
+  repository?: MonitorRepository;
+  providers?: MonitorProviderAdapter[];
+}): MonitorService => {
+  const providersById = new Map<MonitorProviderId, MonitorProviderAdapter>(
+    providers.map((provider) => [provider.providerId, provider]),
+  );
+
+  const resolveProvider = (providerId: MonitorProviderId): MonitorProviderAdapter => {
+    const provider = providersById.get(providerId);
+    if (!provider) {
+      throw new MonitorInputError(`Unsupported monitor provider: ${providerId}`);
+    }
+
+    return provider;
+  };
+
+  return {
+    async readConfig() {
+      const config = repository.readConfig();
+      return sanitizeConfig(config, providersById);
+    },
+
+    async patchConfig(patch: MonitorConfigPatchInput) {
+      const now = new Date();
+      const config = repository.readConfig();
+
+      if (patch.providerId) {
+        config.providerId = patch.providerId;
+      }
+
+      if (patch.queryTerms) {
+        const queryTerms = normalizeQueryTerms(patch.queryTerms);
+        if (queryTerms.length === 0) {
+          throw new MonitorInputError("At least one monitor query term is required.");
+        }
+
+        config.queryTerms = queryTerms;
+      }
+
+      if (patch.refreshPolicy?.maxCacheAgeMs !== undefined) {
+        const maxCacheAgeMs = Math.floor(patch.refreshPolicy.maxCacheAgeMs);
+        if (!Number.isFinite(maxCacheAgeMs) || maxCacheAgeMs <= 0) {
+          throw new MonitorInputError("refreshPolicy.maxCacheAgeMs must be a positive number.");
+        }
+        config.refreshPolicy.maxCacheAgeMs = maxCacheAgeMs;
+      }
+
+      if (patch.refreshPolicy?.maxPosts !== undefined) {
+        const maxPosts = Math.floor(patch.refreshPolicy.maxPosts);
+        if (!Number.isFinite(maxPosts) || maxPosts <= 0) {
+          throw new MonitorInputError("refreshPolicy.maxPosts must be a positive number.");
+        }
+        config.refreshPolicy.maxPosts = maxPosts;
+      }
+
+      if (patch.refreshPolicy?.searchWindowDays !== undefined) {
+        const searchWindowDays = Math.floor(patch.refreshPolicy.searchWindowDays);
+        if (searchWindowDays !== 1 && searchWindowDays !== 3 && searchWindowDays !== 7) {
+          throw new MonitorInputError("refreshPolicy.searchWindowDays must be one of: 1, 3, 7.");
+        }
+        config.refreshPolicy.searchWindowDays = searchWindowDays;
+      }
+
+      if (patch.credentials !== undefined) {
+        const provider = resolveProvider(config.providerId);
+        let credentials: unknown;
+        try {
+          ({ credentials } = provider.saveCredentials(patch.credentials, now));
+        } catch (error) {
+          const message =
+            error instanceof Error ? error.message : "Unable to save monitor credentials.";
+          throw new MonitorInputError(message);
+        }
+
+        if (patch.validateCredentials !== false) {
+          const validation = await provider.validateCredentials(credentials);
+          if (!validation.ok) {
+            throw new MonitorInputError(validation.error ?? "Monitor credentials are invalid.");
+          }
+        }
+
+        config.providers.x.credentials =
+          credentials as PersistedMonitorConfig["providers"]["x"]["credentials"];
+      }
+
+      repository.writeConfig(config);
+      return sanitizeConfig(config, providersById);
+    },
+
+    async readFeed(options: MonitorReadFeedOptions = {}) {
+      const now = new Date();
+      const config = repository.readConfig();
+      const cache = repository.readCache();
+      const provider = resolveProvider(config.providerId);
+
+      const shouldRefresh =
+        options.forceRefresh === true ||
+        (options.refreshIfStale !== false &&
+          isMonitorCacheStale({
+            now,
+            maxCacheAgeMs: config.refreshPolicy.maxCacheAgeMs,
+            lastFetchedAt: cache.fetchedAt,
+            cachedQueryTerms: cache.queryTerms,
+            currentQueryTerms: config.queryTerms,
+          }));
+
+      if (!shouldRefresh) {
+        return toFeedSnapshot(config, cache, now);
+      }
+
+      if (!config.providers.x.credentials) {
+        const nextCache: PersistedMonitorCache = {
+          ...cache,
+          providerId: config.providerId,
+          queryTerms: [...config.queryTerms],
+          lastError: "X credentials are not configured.",
+        };
+        repository.writeCache(nextCache);
+        return toFeedSnapshot(config, nextCache, now);
+      }
+
+      if (config.queryTerms.length === 0) {
+        const nextCache: PersistedMonitorCache = {
+          ...cache,
+          providerId: config.providerId,
+          queryTerms: [...config.queryTerms],
+          lastError: "At least one monitor query term is required.",
+        };
+        repository.writeCache(nextCache);
+        return toFeedSnapshot(config, nextCache, now);
+      }
+
+      try {
+        const [posts, usage] = await Promise.all([
+          provider.fetchRecentPosts({
+            credentials: config.providers.x.credentials,
+            queryTerms: config.queryTerms,
+            postLimit: config.refreshPolicy.maxPosts,
+            searchWindowDays: config.refreshPolicy.searchWindowDays,
+            now,
+          }),
+          provider.fetchUsage({
+            credentials: config.providers.x.credentials,
+            now,
+          }),
+        ]);
+
+        const nextCache: PersistedMonitorCache = {
+          version: 1,
+          providerId: config.providerId,
+          queryTerms: [...config.queryTerms],
+          fetchedAt: now.toISOString(),
+          lastError: null,
+          posts: rankAndLimitPostsByLikes(posts, config.refreshPolicy.maxPosts),
+          usage,
+        };
+
+        repository.writeCache(nextCache);
+        return toFeedSnapshot(config, nextCache, now);
+      } catch (error) {
+        const message = error instanceof Error ? error.message : "Unable to refresh monitor feed.";
+        const nextCache: PersistedMonitorCache = {
+          ...cache,
+          providerId: config.providerId,
+          queryTerms: [...config.queryTerms],
+          lastError: message,
+        };
+
+        repository.writeCache(nextCache);
+        return toFeedSnapshot(config, nextCache, now);
+      }
+    },
+  };
+};