microfox 1.0.11

package/dist/background-agent/eslint.config.js.txt

@@ -0,0 +1,48 @@
+import js from '@eslint/js';
+import tseslint from '@typescript-eslint/eslint-plugin';
+import tsparser from '@typescript-eslint/parser';
+
+export default [
+  js.configs.recommended,
+  {
+    files: ['**/*.ts', '**/*.tsx'],
+    languageOptions: {
+      parser: tsparser,
+      parserOptions: {
+        ecmaVersion: 2020,
+        sourceType: 'module',
+        ecmaFeatures: {
+          jsx: true,
+        },
+      },
+      globals: {
+        console: 'readonly',
+        process: 'readonly',
+      },
+    },
+    plugins: {
+      '@typescript-eslint': tseslint,
+    },
+    rules: {
+      ...tseslint.configs.recommended.rules,
+      'no-unused-vars': 'off',
+      '@typescript-eslint/no-unused-vars': ['warn'],
+      'no-console': 'warn',
+      'react/prop-types': 'off',
+      'no-case-declarations': 'off',
+      '@typescript-eslint/no-explicit-any': 'off',
+      '@typescript-eslint/ban-ts-comment': 'off',
+      '@typescript-eslint/no-empty-function': 'off',
+    },
+  },
+  {
+    ignores: [
+      'dist/**',
+      'node_modules/**',
+      '.turbo/**',
+      'coverage/**',
+      '**/*.js',
+      '**/*.test.ts',
+    ],
+  },
+];

package/dist/background-agent/microfox.json.txt

@@ -0,0 +1,4 @@
+{
+  "stage": "prod",
+  "ignored": [".build/**", "package-lock.json"]
+}

package/dist/background-agent/openapi.md.txt

@@ -0,0 +1 @@
+This is a simple API for a Reddit RAG (Retrieval-Augmented Generation) agent. It allows you to index subreddits and perform searches on the indexed content to get AI-generated reports and insights.

package/dist/background-agent/package.json.txt

@@ -0,0 +1,47 @@
+{
+  "name": "<%= agentName %>",
+  "version": "1.0.0",
+  "description": "A Microbizz agent.",
+  "main": "dist/index.js",
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "deploy": "npm run build && serverless deploy --stage dev",
+    "deploy:preview": "npm run build && serverless deploy --stage preview",
+    "deploy:prod": "npm run build && serverless deploy --stage prod",
+    "remove": "serverless remove",
+    "remove:preview": "serverless remove --stage preview",
+    "remove:prod": "serverless remove --stage prod",
+    "start": "serverless offline start",
+    "dev": "npm run build && serverless offline start",
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix"
+  },
+  "dependencies": {
+    "@aws-sdk/client-sqs": "^3.504.0",
+    "@microfox/ai": "^1.2.0",
+    "@microfox/ai-provider-anthropic": "^1.3.0",
+    "@microfox/db-upstash": "^1.1.4",
+    "@microfox/rag-upstash": "^1.1.1",
+    "@microfox/tool-core": "^1.0.6",
+    "@upstash/redis": "^1.28.4",
+    "ai": "latest",
+    "aws-lambda": "*",
+    "dotenv": "^16.4.5",
+    "uuid": "^9.0.1"
+  },
+  "devDependencies": {
+    "@eslint/js": "latest",
+    "@types/aws-lambda": "^8.10.138",
+    "@types/node": "^20.11.19",
+    "@types/uuid": "^9.0.8",
+    "@typescript-eslint/eslint-plugin": "latest",
+    "@typescript-eslint/parser": "latest",
+    "eslint": "^9.1.0",
+    "serverless": "^3.38.0",
+    "serverless-offline": "^13.3.3",
+    "serverless-plugin-typescript": "^2.1.5",
+    "serverless-step-functions": "^3.23.2",
+    "typescript": "^5.3.3"
+  }
+}

package/dist/background-agent/serverless.yml.txt

@@ -0,0 +1,123 @@
+service: <%= agentName %>
+
+package:
+  excludeDevDependencies: true
+  exclude:
+    - venv/**
+    - .idea/**
+    - .vscode/**
+    - src/**
+    - node_modules/serverless-offline/**
+
+custom:
+  stage: ${env:ENVIRONMENT, 'dev'}
+  serverless-offline:
+    httpPort: 4000
+    lambdaPort: 4002
+    useChildProcesses: true
+    useWorkerThreads: true
+    noCookieValidation: true
+    allowCache: true
+    hideStackTraces: false
+    disableCookieValidation: true
+    noTimeout: true
+    environment: ${file(env.json)}
+
+provider:
+  name: aws
+  runtime: nodejs20.x
+  region: us-east-1
+  stage: ${env:ENVIRONMENT, 'dev'}
+  environment: ${file(env.json)}
+  iam:
+    role:
+      statements:
+        - Effect: "Allow"
+          Action:
+            - "sqs:SendMessage"
+          Resource:
+            - Fn::GetAtt: [IndexQueue, Arn]
+            - Fn::GetAtt: [PaginateQueue, Arn]
+
+plugins:
+  - serverless-plugin-typescript
+  - serverless-offline
+  - serverless-step-functions
+
+functions:
+  # Cron job to populate the index queue
+  cronPopulate:
+    handler: dist/functions/cron-populate.handler
+    environment:
+      INDEX_QUEUE_URL:
+        Ref: IndexQueue
+    events:
+      - schedule: rate(1 day)
+
+  # Cron job to paginate
+  cronPaginate:
+    handler: dist/functions/cron-paginate.handler
+    environment:
+      PAGINATE_QUEUE_URL:
+        Ref: PaginateQueue
+    events:
+      - sqs:
+          arn:
+            Fn::GetAtt:
+              - PaginateQueue
+              - Arn
+
+  # HTTP trigger to add a URL to the index queue
+  triggerIndex:
+    handler: dist/functions/route-trigger-index.handler
+    environment:
+      INDEX_QUEUE_URL:
+        Ref: IndexQueue
+    events:
+      - http:
+          path: /index
+          method: POST
+          cors: true
+
+  # SQS consumer to index a URL
+  sqsIndex:
+    handler: dist/functions/sqs-index.handler
+    events:
+      - sqs:
+          arn:
+            Fn::GetAtt:
+              - IndexQueue
+              - Arn
+
+  # Documentation
+  getDocs:
+    handler: dist/index.getDocs
+    events:
+      - http:
+          path: /docs.json
+          method: get
+          cors: true
+
+resources:
+  Resources:
+    IndexQueue:
+      Type: AWS::SQS::Queue
+      Properties:
+        QueueName: ${self:service}-index-queue-${opt:stage,env:ENVIRONMENT, 'dev'}
+        VisibilityTimeout: 300
+    PaginateQueue:
+      Type: AWS::SQS::Queue
+      Properties:
+        QueueName: ${self:service}-paginate-queue-${opt:stage,env:ENVIRONMENT, 'dev'}
+        VisibilityTimeout: 300
+  Outputs:
+    ApiEndpoints:
+      Description: "API Endpoints"
+      Value:
+        Fn::Join:
+          - ""
+          - - "API: https://"
+            - Ref: "ApiGatewayRestApi"
+            - ".execute-api."
+            - Ref: "AWS::Region"
+            - ".amazonaws.com/${env:ENVIRONMENT, 'dev'}"

package/dist/background-agent/src/functions/cron-paginate.ts.txt

@@ -0,0 +1,135 @@
+import { SQSClient, SendMessageCommand } from "@aws-sdk/client-sqs";
+import {
+  subredditStore,
+  ragSubredditPaginator,
+  subredditListPaginator,
+} from "../helpers/ragRedis.js";
+import { handler as indexSubredditHandler } from "./sqs-index.js";
+import { createMockSQSEvent, ProcessTask } from "@microfox/tool-core";
+
+const sqsClient = new SQSClient({});
+const isOffline = process.env.IS_OFFLINE === "true";
+const STALE_AFTER_SECONDS = 60 * 60; // 1 hour
+const ITEMS_PER_PAGE = 100;
+
+/**
+ * An example of how a cronjob indexes subreddits progressively (in batches)
+ * @returns
+ */
+export const handler = async (): Promise<void> => {
+  console.log("handler: cron-index-subreddit");
+
+  const taskHandler = new ProcessTask({
+    url: process.env.TASK_UPSTASH_REDIS_REST_URL,
+    token: process.env.TASK_UPSTASH_REDIS_REST_TOKEN,
+  });
+
+  try {
+    let paginatorState = await subredditListPaginator.getCurrentStatus();
+    if (!paginatorState) {
+      paginatorState = await subredditListPaginator.startNewIndexing({
+        page: 1,
+        itemsPerPage: ITEMS_PER_PAGE,
+        totalCount: 0,
+      });
+    }
+    if (!paginatorState.progress) {
+      console.error("handler: paginator state is not valid", {
+        paginatorState,
+      });
+      return;
+    }
+    const page = paginatorState.progress.page;
+    console.log("handler: processing page", { page });
+
+    const allSubreddits = await subredditStore.list();
+    const totalCount = allSubreddits.length;
+    const totalPages = Math.ceil(totalCount / ITEMS_PER_PAGE);
+    console.log("handler: subreddits stats", { totalCount, totalPages });
+
+    if (totalCount === 0) {
+      console.log("handler: no subreddits to process");
+      return;
+    }
+
+    const startIndex = (page - 1) * ITEMS_PER_PAGE;
+    if (startIndex >= totalCount && totalCount > 0) {
+      console.log("handler: page out of bounds, resetting to page 1");
+      await subredditListPaginator.updateIndexingStatus({
+        page: 1,
+        totalCount,
+      });
+      // We could either process page 1 now or wait for the next cron run.
+      // Let's wait for the next run to keep logic simple.
+      return;
+    }
+
+    const subredditsToProcess = allSubreddits.slice(
+      startIndex,
+      startIndex + ITEMS_PER_PAGE
+    );
+    console.log("handler: processing subreddits", {
+      count: subredditsToProcess.length,
+    });
+
+    for (const subreddit of subredditsToProcess) {
+      const subredditName = subreddit.name;
+      console.log("handler: checking subreddit", { subredditName });
+
+      const isStale =
+        await ragSubredditPaginator(subredditName).isStale(STALE_AFTER_SECONDS);
+
+      if (isStale) {
+        console.log("handler: subreddit is stale, triggering indexing", {
+          subredditName,
+        });
+
+        const task = await taskHandler.createTask({
+          subreddit: subredditName,
+        });
+
+        const sqsMessageBody = {
+          subreddit: subredditName,
+          task_id: task.id,
+        };
+
+        if (isOffline) {
+          console.log(
+            "handler: running in offline mode, invoking handler directly"
+          );
+          const sqsEvent = createMockSQSEvent(sqsMessageBody);
+          await indexSubredditHandler(sqsEvent);
+          console.log("handler: invoked handler directly");
+        } else {
+          if (!process.env.INDEX_SUBREDDIT_QUEUE_URL) {
+            console.error("INDEX_SUBREDDIT_QUEUE_URL is not set");
+            continue; // Move to the next subreddit
+          }
+          console.log("handler: sending message to SQS", {
+            queueUrl: process.env.INDEX_SUBREDDIT_QUEUE_URL,
+          });
+          const command = new SendMessageCommand({
+            QueueUrl: process.env.INDEX_SUBREDDIT_QUEUE_URL,
+            MessageBody: JSON.stringify(sqsMessageBody),
+          });
+
+          await sqsClient.send(command);
+          console.log("handler: sent message to SQS");
+        }
+      } else {
+        console.log("handler: subreddit is not stale, skipping", {
+          subredditName,
+        });
+      }
+    }
+
+    const nextPage = page >= totalPages && totalPages > 0 ? 1 : page + 1;
+    console.log("handler: updating paginator for next run", { nextPage });
+    await subredditListPaginator.updateIndexingStatus({
+      page: nextPage,
+      totalCount,
+    });
+  } catch (error) {
+    console.error("handler: failed to process cron-index-subreddit", error);
+  }
+};

package/dist/background-agent/src/functions/cron-populate.ts.txt

@@ -0,0 +1,33 @@
+import { ToolParse } from "@microfox/tool-core";
+import { subredditStore, SubredditInfo } from "../helpers/ragRedis.js";
+
+const toolHandler = new ToolParse({});
+
+export const handler = async (): Promise<void> => {
+  console.log("cron-save-handler: cron-save-subreddit");
+
+  try {
+    if (!process.env.REDDIT_ACCESS_TOKEN) {
+      console.log("cron-save-handler: fetching env vars for reddit");
+      await toolHandler.fetchEnvVars({
+        stage: "staging",
+        packageName: "@microfox/reddit",
+        templateType: "testing",
+      });
+      console.log("cron-save-handler: fetched env vars for reddit");
+    }
+    if (!process.env.ANTHROPIC_API_KEY) {
+      console.log("cron-save-handler: fetching env vars for anthropic");
+      await toolHandler.fetchEnvVars({
+        stage: "staging",
+        packageName: "@microfox/ai-provider-anthropic",
+        templateType: "testing",
+      });
+      console.log("cron-save-handler: fetched env vars for anthropic");
+    }
+    // TODO: Populate the content
+    const savedSubreddits = new Set<string>();
+  } catch (error) {
+    console.error("Failed to run cron to save subreddits", error);
+  }
+};

package/dist/background-agent/src/functions/route-trigger-index.ts.txt

@@ -0,0 +1,157 @@
+import {
+  APIGatewayProxyEvent,
+  APIGatewayProxyResult,
+  SQSEvent,
+} from "aws-lambda";
+import { SQSClient, SendMessageCommand } from "@aws-sdk/client-sqs";
+import { handler as indexSubredditHandler } from "./sqs-index.js";
+import { createMockSQSEvent, ProcessTask } from "@microfox/tool-core";
+import { ragSubredditPaginator } from "../helpers/ragRedis.js";
+
+const sqsClient = new SQSClient({});
+const isOffline = process.env.IS_OFFLINE === "true";
+
+export const handler = async (
+  event: APIGatewayProxyEvent
+): Promise<APIGatewayProxyResult> => {
+  console.log("handler: sqs-trigger-subreddit", {
+    isOffline,
+    body: event.body,
+  });
+
+  const taskHandler = new ProcessTask({
+    url: process.env.TASK_UPSTASH_REDIS_REST_URL,
+    token: process.env.TASK_UPSTASH_REDIS_REST_TOKEN,
+  });
+
+  if (!event.body) {
+    console.log("handler: missing request body");
+    return {
+      statusCode: 400,
+      body: JSON.stringify({ message: "Missing request body" }),
+    };
+  }
+
+  try {
+    console.log("handler: parsing request body");
+    const { subreddit } = JSON.parse(event.body);
+    console.log("handler: parsed request body", { subreddit });
+    if (!subreddit) {
+      console.log("handler: missing subreddit in request body");
+      return {
+        statusCode: 400,
+        body: JSON.stringify({ message: "Missing subreddit in request body" }),
+      };
+    }
+
+    if (!process.env.INDEX_SUBREDDIT_QUEUE_URL && !isOffline) {
+      console.error("INDEX_SUBREDDIT_QUEUE_URL is not set");
+      return {
+        statusCode: 500,
+        body: JSON.stringify({
+          message: "Internal server error: Queue not configured",
+        }),
+      };
+    }
+
+    // TODO: get the subreddit info
+    // TODO: store/update it in db
+
+    const task = await taskHandler.createTask({
+      subreddit,
+    });
+
+    console.log("handler: preparing SQS message");
+    const authHeader = event.headers["x-auth-secrets"];
+    const sqsMessageBody = { subreddit, xAuthSecrets: authHeader, task_id: task.id};
+    console.log("handler: prepared SQS message", { sqsMessageBody });
+
+    console.log("handler: checking if subreddit is stale", { subreddit });
+    const isStale = await ragSubredditPaginator(subreddit).isStale(60 * 60);
+    console.log("handler: checked if subreddit is stale", { isStale });
+
+    if (!isStale) {
+      console.log("handler: subreddit already indexed in the last hour");
+      return {
+        statusCode: 202,
+        body: JSON.stringify({
+          message: "Subreddit already indexed in the last hour",
+        }),
+      };
+    }
+
+    if (isOffline) {
+      console.log(
+        "handler: running in offline mode, invoking handler directly"
+      );
+      const sqsEvent = createMockSQSEvent(sqsMessageBody);
+      indexSubredditHandler(sqsEvent);
+      console.log("handler: invoked handler directly");
+    } else {
+      console.log("handler: sending message to SQS", {
+        queueUrl: process.env.INDEX_SUBREDDIT_QUEUE_URL,
+      });
+      const command = new SendMessageCommand({
+        QueueUrl: process.env.INDEX_SUBREDDIT_QUEUE_URL,
+        MessageBody: JSON.stringify(sqsMessageBody),
+      });
+
+      await sqsClient.send(command);
+      console.log("handler: sent message to SQS");
+    }
+
+    console.log("handler: finished successfully");
+    return {
+      statusCode: 202,
+      body: JSON.stringify({
+        message: "Subreddit indexing triggered successfully",
+      }),
+    };
+  } catch (error) {
+    console.error(error);
+    if (error instanceof SyntaxError) {
+      console.log("handler: invalid JSON in request body");
+      return {
+        statusCode: 400,
+        body: JSON.stringify({ message: "Invalid JSON in request body" }),
+      };
+    }
+    console.log("handler: internal server error");
+    return {
+      statusCode: 500,
+      body: JSON.stringify({ message: "Internal server error" }),
+    };
+  }
+};
+
+export const handleApiRequestDocs = {
+  summary: "Trigger subreddit indexing",
+  description:
+    "Triggers the indexing of a specific subreddit by sending a message to the SQS queue.",
+  tags: ["Subreddit"],
+  requestBody: {
+    required: true,
+    content: {
+      "application/json": {
+        schema: {
+          type: "object",
+          properties: {
+            subreddit: {
+              type: "string",
+              description: "The name of the subreddit to index.",
+            },
+          },
+          required: ["subreddit"],
+        },
+      },
+    },
+  },
+  responses: {
+    "202": {
+      description:
+        "Accepted: The indexing request has been successfully queued.",
+    },
+    "400": { description: "Bad Request: Missing or invalid request body." },
+    "500": { description: "Internal server error." },
+  },
+};