@rharkor/caching-for-turbo 2.0.0 → 2.1.0

package/src/cli.ts

@@ -1,94 +0,0 @@
-#!/usr/bin/env node
-
-import { config } from 'dotenv'
-import { server } from './lib/server'
-import { killServer, launchServer } from './lib/server/utils'
-import { logger } from '@rharkor/logger'
-import { ping } from './lib/ping'
-import { serverLogFile } from './lib/constants'
-import { writeFile } from 'fs/promises'
-import { version } from '../package.json'
-
-// Load environment variables from .env file if it exists
-config()
-
-const main = async (): Promise<void> => {
-  await logger.init()
-
-  const args = process.argv.slice(2)
-
-  if (args.includes('--help') || args.includes('-h')) {
-    console.log(`
-Turborepo GitHub Actions Cache Server
-
-Usage:
-  turbogha [options]
-
-Options:
-  --server          Run the server in foreground mode
-  --help, -h        Show this help message
-  --version, -v     Show version
-  --ping            Ping the server
-  --kill            Kill the server
-
-Environment Variables:
-  The following environment variables are supported for S3 configuration:
-  - S3_ACCESS_KEY_ID: AWS S3 access key ID
-  - S3_SECRET_ACCESS_KEY: AWS S3 secret access key  
-  - S3_BUCKET: AWS S3 bucket name
-  - S3_REGION: AWS S3 region
-  - S3_ENDPOINT: S3 endpoint (default: https://s3.amazonaws.com)
-  - S3_PREFIX: Prefix for S3 objects (default: turbogha/)
-  - PROVIDER: Cache provider (github or s3)
-
-Examples:
-  # Run server in background and export environment variables
-  turbogha
-
-  # Run server in foreground mode
-  turbogha --server
-
-  # Ping the server
-  turbogha --ping
-
-  # With S3 configuration
-  S3_ACCESS_KEY_ID=your-key S3_SECRET_ACCESS_KEY=your-secret S3_BUCKET=your-bucket S3_REGION=us-east-1 turbogha
-`)
-    process.exit(0)
-  }
-
-  if (args.includes('--version') || args.includes('-v')) {
-    console.log(version)
-    process.exit(0)
-  }
-
-  try {
-    if (args.includes('--ping')) {
-      await ping()
-    } else if (args.includes('--kill')) {
-      await killServer()
-    } else if (args.includes('--server')) {
-      // Empty log file
-      await writeFile(serverLogFile, '', { flag: 'w' })
-      // Run server in foreground mode
-      console.log('Starting Turborepo cache server in foreground mode...')
-      await server()
-    } else {
-      // Run server in background mode and export environment variables
-      console.log('Starting Turborepo cache server...')
-      // Empty log file
-      await writeFile(serverLogFile, '', { flag: 'w' })
-      await launchServer()
-      console.log(
-        '\nServer is running! You can now use Turbo with remote caching.'
-      )
-      console.log('\nTo stop the server, run:')
-      console.log('curl -X DELETE http://localhost:41230/shutdown')
-    }
-  } catch (error) {
-    console.error('Error:', error)
-    process.exit(1)
-  }
-}
-
-main()

package/src/dev/cleanup.ts

@@ -1,14 +0,0 @@
-import { config } from 'dotenv'
-config()
-
-import { cleanup } from 'src/lib/server/cleanup'
-
-const main = async () => {
-  await cleanup({
-    log: {
-      info: console.log
-    }
-  })
-}
-
-main()

package/src/dev-run.ts

@@ -1,16 +0,0 @@
-// Run the server in foreground and kill it after the test
-
-import { config } from 'dotenv'
-config()
-
-import { server } from './lib/server'
-import { launchServer } from './lib/server/utils'
-
-const main = async (): Promise<void> => {
-  //* Run server
-  server()
-  //* Run launch server
-  await launchServer(true)
-}
-
-main()

package/src/index.ts

@@ -1,6 +0,0 @@
-/**
- * The entrypoint for the action.
- */
-import { run } from './main'
-
-run()

package/src/lib/constants.ts

@@ -1,29 +0,0 @@
-import { join } from 'path'
-import { env } from './env'
-import { core } from './core'
-
-// Helper function to get input value, prioritizing environment variables for local development
-const getInput = (name: string, envName?: string): string | undefined => {
-  // In GitHub Actions context, try core.getInput first
-  if (process.env.CI === 'true') {
-    const coreInput = core.getInput(name)
-    if (coreInput) return coreInput
-  }
-
-  // Fall back to environment variable
-  const envVar = envName || name.toUpperCase().replace(/-/g, '_')
-  return process.env[envVar]
-}
-
-export const serverPort = 41230
-export const cachePath = 'turbogha_'
-export const cachePrefix = getInput('cache-prefix', 'CACHE_PREFIX') || cachePath
-export const getCacheKey = (hash: string, tag?: string): string =>
-  `${cachePrefix}${hash}${tag ? `#${tag}` : ''}`
-export const serverLogFile = env.RUNNER_TEMP
-  ? join(env.RUNNER_TEMP, 'turbogha.log')
-  : '/tmp/turbogha.log'
-export const getFsCachePath = (hash: string): string =>
-  join(env.RUNNER_TEMP || '/tmp', `${hash}.tg.bin`)
-export const getTempCachePath = (key: string): string =>
-  join(env.RUNNER_TEMP || '/tmp', `cache-${key}.tg.bin`)

package/src/lib/core.ts

@@ -1,62 +0,0 @@
-import * as coreLib from '@actions/core'
-import { logger as loggerLib } from '@rharkor/logger'
-
-const isCI = process.env.CI === 'true'
-
-export const core = {
-  isCI,
-  setFailed: (message: string) => {
-    if (isCI) {
-      coreLib.setFailed(message)
-    } else {
-      loggerLib.error(message)
-    }
-  },
-  getInput: (name: string) => {
-    if (isCI) {
-      return coreLib.getInput(name)
-    }
-    return undefined
-  },
-  exportVariable: (name: string, value: string) => {
-    if (isCI) {
-      coreLib.exportVariable(name, value)
-    }
-  },
-  //* Logger
-  info: (message: string) => {
-    if (isCI) {
-      coreLib.info(message)
-    } else {
-      loggerLib.info(message)
-    }
-  },
-  error: (message: string) => {
-    if (isCI) {
-      coreLib.error(message)
-    } else {
-      loggerLib.error(message)
-    }
-  },
-  debug: (message: string) => {
-    if (isCI) {
-      coreLib.debug(message)
-    } else {
-      loggerLib.debug(message)
-    }
-  },
-  log: (message: string) => {
-    if (isCI) {
-      coreLib.info(message)
-    } else {
-      loggerLib.log(message)
-    }
-  },
-  success: (message: string) => {
-    if (isCI) {
-      coreLib.info(message)
-    } else {
-      loggerLib.success(message)
-    }
-  }
-}

package/src/lib/env/index.ts

@@ -1,22 +0,0 @@
-const envObject = {
-  ACTIONS_RUNTIME_TOKEN: process.env.ACTIONS_RUNTIME_TOKEN,
-  ACTIONS_CACHE_URL: process.env.ACTIONS_CACHE_URL,
-  RUNNER_TEMP: process.env.RUNNER_TEMP
-}
-
-type TInvalidEnv = {
-  valid: false
-} & typeof envObject
-
-type TValidEnv = {
-  valid: true
-} & {
-  [K in keyof typeof envObject]: NonNullable<(typeof envObject)[K]>
-}
-
-type TEnv = TInvalidEnv | TValidEnv
-
-export const env = {
-  valid: Object.values(envObject).every(value => value !== undefined),
-  ...envObject
-} as TEnv

package/src/lib/ping.ts

@@ -1,7 +0,0 @@
-import { core } from './core'
-import { waitForServer } from './server/utils'
-
-export const ping = async () => {
-  await waitForServer()
-  core.success('Server is up and running')
-}

package/src/lib/providers/cache/index.ts

@@ -1,87 +0,0 @@
-import { Readable } from 'node:stream'
-import { env } from '../../env'
-import { pipeline } from 'node:stream/promises'
-import {
-  createReadStream,
-  createWriteStream,
-  existsSync,
-  statSync
-} from 'node:fs'
-import { getCacheKey, getFsCachePath, getTempCachePath } from '../../constants'
-import { RequestContext } from '../../server'
-import { TListFile } from '../../server/cleanup'
-import { getCacheClient } from './utils'
-import { TProvider } from '../../providers'
-import { core } from 'src/lib/core'
-
-//* Cache API
-export async function saveCache(
-  ctx: RequestContext,
-  hash: string,
-  tag: string,
-  stream: Readable
-): Promise<void> {
-  if (!env.valid) {
-    ctx.log.info(
-      `Using filesystem cache because cache API env vars are not set`
-    )
-    await pipeline(stream, createWriteStream(getFsCachePath(hash)))
-    return
-  }
-  const client = getCacheClient()
-  const key = getCacheKey(hash, tag)
-  await client.save(key, stream)
-  ctx.log.info(`Saved cache ${key} for ${hash}`)
-}
-
-export async function getCache(
-  ctx: RequestContext,
-  hash: string
-): Promise<
-  [number | undefined, Readable | ReadableStream, string | undefined] | null
-> {
-  //* Get cache from filesystem if cache API env vars are not set
-  if (!env.valid) {
-    const path = getFsCachePath(hash)
-    if (!existsSync(path)) return null
-    const size = statSync(path).size
-    return [size, createReadStream(path), undefined]
-  }
-  //* Get cache from cache API
-  const client = getCacheClient()
-  const cacheKey = getCacheKey(hash)
-  const fileRestorationPath = getTempCachePath(cacheKey)
-  const foundKey = await client.restore(fileRestorationPath, cacheKey)
-  ctx.log.info(`Cache lookup for ${cacheKey}`)
-  if (!foundKey) {
-    ctx.log.info(`Cache lookup did not return data`)
-    return null
-  }
-  const [foundCacheKey, artifactTag] = String(foundKey).split('#')
-  if (foundCacheKey !== cacheKey) {
-    ctx.log.info(`Cache key mismatch: ${foundCacheKey} !== ${cacheKey}`)
-    return null
-  }
-  const size = statSync(fileRestorationPath).size
-  const readableStream = createReadStream(fileRestorationPath)
-  return [size, readableStream, artifactTag]
-}
-
-export async function deleteCache(): Promise<void> {
-  core.error(`Cannot delete github cache automatically.`)
-  throw new Error(`Cannot delete github cache automatically.`)
-}
-
-export async function listCache(): Promise<TListFile[]> {
-  core.error(`Cannot list github cache automatically.`)
-  throw new Error(`Cannot list github cache automatically.`)
-}
-
-export const getGithubProvider = (): TProvider => {
-  return {
-    save: saveCache,
-    get: getCache,
-    delete: deleteCache,
-    list: listCache
-  }
-}

package/src/lib/providers/cache/utils.ts

@@ -1,70 +0,0 @@
-import { Readable } from 'node:stream'
-import { env } from '../../env'
-import streamToPromise from 'stream-to-promise'
-import { createWriteStream } from 'node:fs'
-import { unlink } from 'node:fs/promises'
-import { getTempCachePath } from '../../constants'
-import { restoreCache, saveCache } from '@actions/cache'
-import { core } from 'src/lib/core'
-class HandledError extends Error {
-  status: number
-  statusText: string
-  data: unknown
-  constructor(status: number, statusText: string, data: unknown) {
-    super(`${status}: ${statusText}`)
-    this.status = status
-    this.statusText = statusText
-    this.data = data
-  }
-}
-
-function handleFetchError(message: string) {
-  return (error: unknown) => {
-    if (error instanceof HandledError) {
-      core.error(`${message}: ${error.status} ${error.statusText}`)
-      core.error(JSON.stringify(error.data))
-      throw error
-    }
-    core.error(`${message}: ${error}`)
-    throw error
-  }
-}
-
-export function getCacheClient() {
-  if (!env.valid) {
-    throw new Error('Cache API env vars are not set')
-  }
-
-  const save = async (key: string, stream: Readable): Promise<void> => {
-    try {
-      //* Create a temporary file to store the cache
-      const tempFile = getTempCachePath(key)
-      const writeStream = createWriteStream(tempFile)
-      await streamToPromise(stream.pipe(writeStream))
-      core.info(`Saved cache to ${tempFile}`)
-
-      core.info(`Saving cache for key: ${key}, path: ${tempFile}`)
-      await saveCache([tempFile], key)
-      core.info(`Saved cache ${key}`)
-
-      //* Remove the temporary file
-      await unlink(tempFile)
-    } catch (error) {
-      handleFetchError('Unable to upload cache')(error)
-    }
-  }
-
-  const restore = async (
-    path: string,
-    key: string
-  ): Promise<string | undefined> => {
-    core.info(`Querying cache for key: ${key}, path: ${path}`)
-
-    return restoreCache([path], key, [])
-  }
-
-  return {
-    save,
-    restore
-  }
-}

package/src/lib/providers/s3/index.ts

@@ -1,237 +0,0 @@
-import { TProvider } from 'src/lib/providers'
-
-import { Readable } from 'stream'
-import { RequestContext } from '../../server'
-import { TListFile } from '../../server/cleanup'
-import {
-  S3Client,
-  GetObjectCommand,
-  DeleteObjectCommand,
-  ListObjectsV2Command
-} from '@aws-sdk/client-s3'
-import { Upload } from '@aws-sdk/lib-storage'
-import { getCacheKey } from 'src/lib/constants'
-import { core } from 'src/lib/core'
-
-// Helper function to get input value, prioritizing environment variables for local development
-const getInput = (name: string, envName?: string): string | undefined => {
-  // In GitHub Actions context, try core.getInput first
-  if (process.env.GITHUB_ACTIONS === 'true') {
-    const coreInput = core.getInput(name)
-    if (coreInput) return coreInput
-  }
-
-  // Fall back to environment variable
-  const envVar = envName || name.toUpperCase().replace(/-/g, '_')
-  return process.env[envVar]
-}
-
-export const getS3Provider = (): TProvider => {
-  const s3AccessKeyId = getInput('s3-access-key-id', 'S3_ACCESS_KEY_ID')
-  const s3SecretAccessKey = getInput(
-    's3-secret-access-key',
-    'S3_SECRET_ACCESS_KEY'
-  )
-  const s3Bucket = getInput('s3-bucket', 'S3_BUCKET')
-  const s3Region = getInput('s3-region', 'S3_REGION')
-  const s3Endpoint =
-    getInput('s3-endpoint', 'S3_ENDPOINT') || 'https://s3.amazonaws.com'
-  const s3Prefix = getInput('s3-prefix', 'S3_PREFIX') || 'turbogha/'
-
-  if (!s3AccessKeyId || !s3SecretAccessKey || !s3Bucket || !s3Region) {
-    throw new Error(
-      'S3 provider requires s3-access-key-id, s3-secret-access-key, s3-bucket, and s3-region. Set these as environment variables or GitHub Actions inputs.'
-    )
-  }
-
-  const s3Client = new S3Client({
-    region: s3Region,
-    endpoint: s3Endpoint,
-    credentials: {
-      accessKeyId: s3AccessKeyId,
-      secretAccessKey: s3SecretAccessKey
-    }
-  })
-
-  const getS3Key = (hash: string, tag?: string) => {
-    const key = getCacheKey(hash, tag)
-    if (s3Prefix) {
-      return `${s3Prefix}${key}`
-    }
-    return key
-  }
-
-  const save = async (
-    ctx: RequestContext,
-    hash: string,
-    tag: string,
-    stream: Readable
-  ): Promise<void> => {
-    const objectKey = getS3Key(hash, tag)
-    console.log({ objectKey, s3Prefix })
-
-    try {
-      // Use the S3 Upload utility which handles multipart uploads for large files
-      const upload = new Upload({
-        client: s3Client,
-        params: {
-          Bucket: s3Bucket,
-          Key: objectKey,
-          Body: stream,
-          ContentType: 'application/octet-stream'
-        }
-      })
-
-      await upload.done()
-      ctx.log.info(`Saved artifact to S3: ${objectKey}`)
-    } catch (error) {
-      ctx.log.info(`Error saving artifact to S3: ${error}`)
-      throw error
-    }
-  }
-
-  const get = async (
-    ctx: RequestContext,
-    hash: string
-  ): Promise<
-    [number | undefined, Readable | ReadableStream, string | undefined] | null
-  > => {
-    // First try to get with just the hash
-    const objectKey = getS3Key(hash)
-
-    try {
-      // Try to find the object
-      const listCommand = new ListObjectsV2Command({
-        Bucket: s3Bucket,
-        Prefix: objectKey,
-        MaxKeys: 10
-      })
-
-      const listResponse = await s3Client.send(listCommand)
-
-      if (!listResponse.Contents || listResponse.Contents.length === 0) {
-        ctx.log.info(`No cached artifact found for ${hash}`)
-        return null
-      }
-
-      // Find the most recent object that matches the hash prefix
-      const matchingObjects = listResponse.Contents.filter(
-        obj => obj.Key && obj.Key.startsWith(objectKey)
-      )
-
-      if (matchingObjects.length === 0) {
-        return null
-      }
-
-      // Sort by last modified date, newest first
-      matchingObjects.sort((a, b) => {
-        const dateA = a.LastModified?.getTime() || 0
-        const dateB = b.LastModified?.getTime() || 0
-        return dateB - dateA
-      })
-
-      const latestObject = matchingObjects[0]
-      const key = latestObject.Key as string
-
-      // Get the object
-      const getCommand = new GetObjectCommand({
-        Bucket: s3Bucket,
-        Key: key
-      })
-
-      const response = await s3Client.send(getCommand)
-
-      if (!response.Body) {
-        ctx.log.info(`Failed to get artifact body from S3`)
-        return null
-      }
-
-      const size = response.ContentLength
-      const stream = response.Body as Readable
-
-      // Extract the tag if it exists
-      let artifactTag: string | undefined
-      if (key.includes('#')) {
-        const parts = key.split('#')
-        artifactTag = parts[parts.length - 1]
-      }
-
-      ctx.log.info(`Retrieved artifact from S3: ${key}`)
-      return [size, stream, artifactTag]
-    } catch (error) {
-      ctx.log.info(`Error getting artifact from S3: ${error}`)
-      return null
-    }
-  }
-
-  const deleteObj = async (key: string): Promise<void> => {
-    try {
-      const deleteCommand = new DeleteObjectCommand({
-        Bucket: s3Bucket,
-        Key: key
-      })
-
-      await s3Client.send(deleteCommand)
-    } catch (error) {
-      core.error(`Error deleting artifact from S3: ${error}`)
-      throw error
-    }
-  }
-
-  const list = async (): Promise<TListFile[]> => {
-    try {
-      const files: TListFile[] = []
-      let continuationToken: string | undefined
-
-      do {
-        // Create a new command for each request with the current continuation token
-        const listCommand = new ListObjectsV2Command({
-          Bucket: s3Bucket,
-          Prefix: s3Prefix,
-          MaxKeys: 1000,
-          ContinuationToken: continuationToken
-        })
-
-        core.debug(
-          `Listing S3 objects with prefix ${s3Prefix}${continuationToken ? ' and continuation token' : ''}`
-        )
-
-        const response = await s3Client.send(listCommand)
-
-        if (response.Contents && response.Contents.length > 0) {
-          core.debug(`Found ${response.Contents.length} objects`)
-
-          const objects = response.Contents.filter(obj => obj.Key).map(
-            (obj): TListFile => {
-              return {
-                path: obj.Key as string,
-                createdAt: (obj.LastModified || new Date()).toISOString(),
-                size: obj.Size || 0
-              }
-            }
-          )
-
-          files.push(...objects)
-        }
-
-        continuationToken = response.NextContinuationToken
-        if (continuationToken) {
-          core.debug(`NextContinuationToken: ${continuationToken}`)
-        }
-      } while (continuationToken)
-
-      core.debug(`Total files listed: ${files.length}`)
-      return files
-    } catch (error) {
-      core.error(`Error listing artifacts from S3: ${error}`)
-      throw error
-    }
-  }
-
-  return {
-    save,
-    get,
-    delete: deleteObj,
-    list
-  }
-}

package/src/lib/providers.ts

@@ -1,42 +0,0 @@
-import { Readable } from 'stream'
-import { TListFile } from './server/cleanup'
-import { RequestContext } from './server'
-import { getGithubProvider } from './providers/cache'
-import { getS3Provider } from './providers/s3'
-import { core } from './core'
-
-export type TProvider = {
-  save: (
-    ctx: RequestContext,
-    hash: string,
-    tag: string,
-    stream: Readable
-  ) => Promise<void>
-  get: (
-    ctx: RequestContext,
-    hash: string
-  ) => Promise<
-    [number | undefined, Readable | ReadableStream, string | undefined] | null
-  >
-  delete: (key: string) => Promise<void>
-  list: () => Promise<TListFile[]>
-}
-
-export const getProvider = (): TProvider => {
-  const provider = core.getInput('provider') || process.env.PROVIDER
-
-  if (!provider) {
-    throw new Error(
-      'Provider is required. Set PROVIDER environment variable or provider input.'
-    )
-  }
-
-  if (provider === 'github') {
-    return getGithubProvider()
-  }
-  if (provider === 's3') {
-    return getS3Provider()
-  }
-
-  throw new Error(`Provider ${provider} not supported`)
-}