@rharkor/caching-for-turbo 2.0.0 → 2.1.1

package/src/lib/providers/s3/index.ts

@@ -1,237 +0,0 @@
-import { TProvider } from 'src/lib/providers'
-
-import { Readable } from 'stream'
-import { RequestContext } from '../../server'
-import { TListFile } from '../../server/cleanup'
-import {
-  S3Client,
-  GetObjectCommand,
-  DeleteObjectCommand,
-  ListObjectsV2Command
-} from '@aws-sdk/client-s3'
-import { Upload } from '@aws-sdk/lib-storage'
-import { getCacheKey } from 'src/lib/constants'
-import { core } from 'src/lib/core'
-
-// Helper function to get input value, prioritizing environment variables for local development
-const getInput = (name: string, envName?: string): string | undefined => {
-  // In GitHub Actions context, try core.getInput first
-  if (process.env.GITHUB_ACTIONS === 'true') {
-    const coreInput = core.getInput(name)
-    if (coreInput) return coreInput
-  }
-
-  // Fall back to environment variable
-  const envVar = envName || name.toUpperCase().replace(/-/g, '_')
-  return process.env[envVar]
-}
-
-export const getS3Provider = (): TProvider => {
-  const s3AccessKeyId = getInput('s3-access-key-id', 'S3_ACCESS_KEY_ID')
-  const s3SecretAccessKey = getInput(
-    's3-secret-access-key',
-    'S3_SECRET_ACCESS_KEY'
-  )
-  const s3Bucket = getInput('s3-bucket', 'S3_BUCKET')
-  const s3Region = getInput('s3-region', 'S3_REGION')
-  const s3Endpoint =
-    getInput('s3-endpoint', 'S3_ENDPOINT') || 'https://s3.amazonaws.com'
-  const s3Prefix = getInput('s3-prefix', 'S3_PREFIX') || 'turbogha/'
-
-  if (!s3AccessKeyId || !s3SecretAccessKey || !s3Bucket || !s3Region) {
-    throw new Error(
-      'S3 provider requires s3-access-key-id, s3-secret-access-key, s3-bucket, and s3-region. Set these as environment variables or GitHub Actions inputs.'
-    )
-  }
-
-  const s3Client = new S3Client({
-    region: s3Region,
-    endpoint: s3Endpoint,
-    credentials: {
-      accessKeyId: s3AccessKeyId,
-      secretAccessKey: s3SecretAccessKey
-    }
-  })
-
-  const getS3Key = (hash: string, tag?: string) => {
-    const key = getCacheKey(hash, tag)
-    if (s3Prefix) {
-      return `${s3Prefix}${key}`
-    }
-    return key
-  }
-
-  const save = async (
-    ctx: RequestContext,
-    hash: string,
-    tag: string,
-    stream: Readable
-  ): Promise<void> => {
-    const objectKey = getS3Key(hash, tag)
-    console.log({ objectKey, s3Prefix })
-
-    try {
-      // Use the S3 Upload utility which handles multipart uploads for large files
-      const upload = new Upload({
-        client: s3Client,
-        params: {
-          Bucket: s3Bucket,
-          Key: objectKey,
-          Body: stream,
-          ContentType: 'application/octet-stream'
-        }
-      })
-
-      await upload.done()
-      ctx.log.info(`Saved artifact to S3: ${objectKey}`)
-    } catch (error) {
-      ctx.log.info(`Error saving artifact to S3: ${error}`)
-      throw error
-    }
-  }
-
-  const get = async (
-    ctx: RequestContext,
-    hash: string
-  ): Promise<
-    [number | undefined, Readable | ReadableStream, string | undefined] | null
-  > => {
-    // First try to get with just the hash
-    const objectKey = getS3Key(hash)
-
-    try {
-      // Try to find the object
-      const listCommand = new ListObjectsV2Command({
-        Bucket: s3Bucket,
-        Prefix: objectKey,
-        MaxKeys: 10
-      })
-
-      const listResponse = await s3Client.send(listCommand)
-
-      if (!listResponse.Contents || listResponse.Contents.length === 0) {
-        ctx.log.info(`No cached artifact found for ${hash}`)
-        return null
-      }
-
-      // Find the most recent object that matches the hash prefix
-      const matchingObjects = listResponse.Contents.filter(
-        obj => obj.Key && obj.Key.startsWith(objectKey)
-      )
-
-      if (matchingObjects.length === 0) {
-        return null
-      }
-
-      // Sort by last modified date, newest first
-      matchingObjects.sort((a, b) => {
-        const dateA = a.LastModified?.getTime() || 0
-        const dateB = b.LastModified?.getTime() || 0
-        return dateB - dateA
-      })
-
-      const latestObject = matchingObjects[0]
-      const key = latestObject.Key as string
-
-      // Get the object
-      const getCommand = new GetObjectCommand({
-        Bucket: s3Bucket,
-        Key: key
-      })
-
-      const response = await s3Client.send(getCommand)
-
-      if (!response.Body) {
-        ctx.log.info(`Failed to get artifact body from S3`)
-        return null
-      }
-
-      const size = response.ContentLength
-      const stream = response.Body as Readable
-
-      // Extract the tag if it exists
-      let artifactTag: string | undefined
-      if (key.includes('#')) {
-        const parts = key.split('#')
-        artifactTag = parts[parts.length - 1]
-      }
-
-      ctx.log.info(`Retrieved artifact from S3: ${key}`)
-      return [size, stream, artifactTag]
-    } catch (error) {
-      ctx.log.info(`Error getting artifact from S3: ${error}`)
-      return null
-    }
-  }
-
-  const deleteObj = async (key: string): Promise<void> => {
-    try {
-      const deleteCommand = new DeleteObjectCommand({
-        Bucket: s3Bucket,
-        Key: key
-      })
-
-      await s3Client.send(deleteCommand)
-    } catch (error) {
-      core.error(`Error deleting artifact from S3: ${error}`)
-      throw error
-    }
-  }
-
-  const list = async (): Promise<TListFile[]> => {
-    try {
-      const files: TListFile[] = []
-      let continuationToken: string | undefined
-
-      do {
-        // Create a new command for each request with the current continuation token
-        const listCommand = new ListObjectsV2Command({
-          Bucket: s3Bucket,
-          Prefix: s3Prefix,
-          MaxKeys: 1000,
-          ContinuationToken: continuationToken
-        })
-
-        core.debug(
-          `Listing S3 objects with prefix ${s3Prefix}${continuationToken ? ' and continuation token' : ''}`
-        )
-
-        const response = await s3Client.send(listCommand)
-
-        if (response.Contents && response.Contents.length > 0) {
-          core.debug(`Found ${response.Contents.length} objects`)
-
-          const objects = response.Contents.filter(obj => obj.Key).map(
-            (obj): TListFile => {
-              return {
-                path: obj.Key as string,
-                createdAt: (obj.LastModified || new Date()).toISOString(),
-                size: obj.Size || 0
-              }
-            }
-          )
-
-          files.push(...objects)
-        }
-
-        continuationToken = response.NextContinuationToken
-        if (continuationToken) {
-          core.debug(`NextContinuationToken: ${continuationToken}`)
-        }
-      } while (continuationToken)
-
-      core.debug(`Total files listed: ${files.length}`)
-      return files
-    } catch (error) {
-      core.error(`Error listing artifacts from S3: ${error}`)
-      throw error
-    }
-  }
-
-  return {
-    save,
-    get,
-    delete: deleteObj,
-    list
-  }
-}

package/src/lib/providers.ts

@@ -1,42 +0,0 @@
-import { Readable } from 'stream'
-import { TListFile } from './server/cleanup'
-import { RequestContext } from './server'
-import { getGithubProvider } from './providers/cache'
-import { getS3Provider } from './providers/s3'
-import { core } from './core'
-
-export type TProvider = {
-  save: (
-    ctx: RequestContext,
-    hash: string,
-    tag: string,
-    stream: Readable
-  ) => Promise<void>
-  get: (
-    ctx: RequestContext,
-    hash: string
-  ) => Promise<
-    [number | undefined, Readable | ReadableStream, string | undefined] | null
-  >
-  delete: (key: string) => Promise<void>
-  list: () => Promise<TListFile[]>
-}
-
-export const getProvider = (): TProvider => {
-  const provider = core.getInput('provider') || process.env.PROVIDER
-
-  if (!provider) {
-    throw new Error(
-      'Provider is required. Set PROVIDER environment variable or provider input.'
-    )
-  }
-
-  if (provider === 'github') {
-    return getGithubProvider()
-  }
-  if (provider === 's3') {
-    return getS3Provider()
-  }
-
-  throw new Error(`Provider ${provider} not supported`)
-}

package/src/lib/server/cleanup.ts

@@ -1,111 +0,0 @@
-import { RequestContext } from '.'
-import parse from 'parse-duration'
-import { getProvider } from '../providers'
-import { parseFileSize } from './utils'
-import { core } from '../core'
-
-export type TListFile = {
-  path: string
-  createdAt: string
-  size: number
-}
-
-export async function cleanup(ctx: RequestContext) {
-  const maxAge = core.getInput('max-age') || process.env.MAX_AGE
-  const maxFiles = core.getInput('max-files') || process.env.MAX_FILES
-  const maxSize = core.getInput('max-size') || process.env.MAX_SIZE
-
-  if (!maxAge && !maxFiles && !maxSize) {
-    ctx.log.info('No cleanup options provided, skipping cleanup')
-    return
-  }
-
-  const { maxAgeParsed, maxFilesParsed, maxSizeParsed } = {
-    maxAgeParsed: maxAge ? parse(maxAge) : undefined,
-    maxFilesParsed: maxFiles ? parseInt(maxFiles) : undefined,
-    maxSizeParsed: maxSize ? parseFileSize(maxSize) : undefined
-  }
-
-  if (maxAge && !maxAgeParsed) {
-    core.error('Invalid max-age provided')
-    throw new Error('Invalid max-age provided')
-  }
-
-  if (maxFiles && !maxFilesParsed) {
-    core.error('Invalid max-files provided')
-    throw new Error('Invalid max-files provided')
-  }
-
-  if (maxSize && !maxSizeParsed) {
-    core.error('Invalid max-size provided')
-    throw new Error('Invalid max-size provided')
-  }
-
-  const provider = getProvider()
-
-  const files = await provider.list()
-
-  const fileToDelete: (TListFile & {
-    reason: 'max-age' | 'max-files' | 'max-size'
-  })[] = []
-  if (maxAgeParsed) {
-    const now = new Date()
-    const age = new Date(now.getTime() - maxAgeParsed)
-    fileToDelete.push(
-      ...files
-        .filter(file => new Date(file.createdAt) < age)
-        .map(file => ({ ...file, reason: 'max-age' as const }))
-    )
-  }
-
-  if (maxFilesParsed && files.length > maxFilesParsed) {
-    const sortedByDate = [...files].sort(
-      (a, b) =>
-        new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime()
-    )
-    const excessFiles = sortedByDate.slice(0, files.length - maxFilesParsed)
-    excessFiles.forEach(file => {
-      if (!fileToDelete.some(f => f.path === file.path)) {
-        fileToDelete.push({ ...file, reason: 'max-files' })
-      }
-    })
-  }
-
-  if (maxSizeParsed) {
-    let totalSize = files.reduce((sum, file) => sum + file.size, 0)
-
-    if (totalSize > maxSizeParsed) {
-      const sortedByDate = [...files].sort(
-        (a, b) =>
-          new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime()
-      )
-
-      for (const file of sortedByDate) {
-        if (totalSize <= maxSizeParsed) break
-
-        if (!fileToDelete.some(f => f.path === file.path)) {
-          fileToDelete.push({ ...file, reason: 'max-size' })
-          totalSize -= file.size
-        }
-      }
-    }
-  }
-
-  if (fileToDelete.length > 0) {
-    ctx.log.info(
-      `Cleaning up ${fileToDelete.length} files (${fileToDelete.map(
-        f => `${f.path} (${f.reason})`
-      )})`
-    )
-    for (const file of fileToDelete) {
-      try {
-        await provider.delete(file.path)
-        ctx.log.info(`Deleted ${file.path}`)
-      } catch (error) {
-        core.error(`Failed to delete ${file.path}: ${error}`)
-      }
-    }
-  } else {
-    ctx.log.info('No files to clean up')
-  }
-}

package/src/lib/server/index.ts

@@ -1,85 +0,0 @@
-import Fastify from 'fastify'
-import { serverPort } from '../constants'
-import { cleanup } from './cleanup'
-import { getProvider } from '../providers'
-
-export type RequestContext = {
-  log: {
-    info: (message: string) => void
-  }
-}
-
-export async function server(): Promise<void> {
-  //* Create the server
-  const fastify = Fastify({
-    logger: true
-  })
-
-  //? Server status check
-  fastify.get('/', async () => {
-    return { ok: true }
-  })
-
-  //? Shut down the server
-  const shutdown = async (ctx: RequestContext) => {
-    //* Handle cleanup
-    await cleanup(ctx)
-
-    // Exit the server after responding (100ms)
-    setTimeout(() => process.exit(0), 100)
-    return { ok: true }
-  }
-  fastify.delete('/shutdown', async request => {
-    return shutdown(request)
-  })
-
-  //? Handle streaming requets body
-  // https://www.fastify.io/docs/latest/Reference/ContentTypeParser/#catch-all
-  fastify.addContentTypeParser(
-    'application/octet-stream',
-    (_req, _payload, done) => {
-      done(null)
-    }
-  )
-
-  //? Upload cache
-  fastify.put('/v8/artifacts/:hash', async request => {
-    const hash = (request.params as { hash: string }).hash
-    request.log.info(`Received artifact for ${hash}`)
-    const provider = getProvider()
-    await provider.save(
-      request,
-      hash,
-      String(request.headers['x-artifact-tag'] || ''),
-      request.raw
-    )
-    request.log.info(`Saved artifact for ${hash}`)
-    return { ok: true }
-  })
-
-  //? Download cache
-  fastify.get('/v8/artifacts/:hash', async (request, reply) => {
-    const hash = (request.params as { hash: string }).hash
-    request.log.info(`Requested artifact for ${hash}`)
-    const provider = getProvider()
-    const result = await provider.get(request, hash)
-    if (result === null) {
-      request.log.info(`Artifact for ${hash} not found`)
-      reply.code(404)
-      return { ok: false }
-    }
-    const [size, stream, artifactTag] = result
-    if (size) {
-      reply.header('Content-Length', size)
-    }
-    reply.header('Content-Type', 'application/octet-stream')
-    if (artifactTag) {
-      reply.header('x-artifact-tag', artifactTag)
-    }
-    request.log.info(`Sending artifact for ${hash}`)
-    return reply.send(stream)
-  })
-
-  //* Start the server
-  await fastify.listen({ port: serverPort })
-}

package/src/lib/server/utils.ts

@@ -1,91 +0,0 @@
-import waitOn from 'wait-on'
-import { cachePrefix, cachePath, serverLogFile, serverPort } from '../constants'
-import { openSync } from 'fs'
-import { spawn } from 'child_process'
-import { core } from '../core'
-
-export const waitForServer = async (): Promise<void> => {
-  await waitOn({
-    resources: [`http-get://localhost:${serverPort}`],
-    timeout: 5000
-  })
-}
-
-export const exportVariable = (name: string, value: string): void => {
-  core.exportVariable(name, value)
-  core.log(`  ${name}=${value}`)
-}
-
-export async function launchServer(devRun?: boolean): Promise<void> {
-  if (!devRun) {
-    //* Launch a detached child process to run the server
-    // See: https://nodejs.org/docs/latest-v16.x/api/child_process.html#optionsdetached
-    const out = openSync(serverLogFile, 'a')
-    const err = openSync(serverLogFile, 'a')
-    const child = spawn(process.argv[0], [process.argv[1], '--server'], {
-      detached: true,
-      stdio: ['ignore', out, err]
-    })
-    child.unref()
-    core.log(`Cache version: ${cachePath}`)
-    core.log(`Cache prefix: ${cachePrefix}`)
-    core.log(`Launched child process: ${child.pid}`)
-    core.log(`Server log file: ${serverLogFile}`)
-  }
-
-  //* Wait for server
-  await waitForServer()
-  core.info(`Server is now up and running.`)
-
-  //* Export the environment variables for Turbo
-  if (devRun) {
-    console.log('Execute:')
-    console.log(`export TURBOGHA_PORT=${serverPort}`)
-    console.log(`export TURBO_API=http://localhost:${serverPort}`)
-    console.log(`export TURBO_TOKEN=turbogha`)
-    console.log(`export TURBO_TEAM=turbogha`)
-  } else {
-    if (core.isCI) {
-      core.info('The following environment variables are exported:')
-    } else {
-      core.info(
-        'You need to use the following environment variables for turbo to work:'
-      )
-    }
-    exportVariable('TURBOGHA_PORT', `${serverPort}`)
-    exportVariable('TURBO_API', `http://localhost:${serverPort}`)
-    exportVariable('TURBO_TOKEN', 'turbogha')
-    exportVariable('TURBO_TEAM', 'turbogha')
-  }
-}
-
-export async function killServer() {
-  //* Kill the server
-  await fetch(`http://localhost:${serverPort}/shutdown`, {
-    method: 'DELETE'
-  })
-}
-
-export const parseFileSize = (size: string): number => {
-  const units: { [key: string]: number } = {
-    b: 1,
-    kb: 1024,
-    mb: 1024 * 1024,
-    gb: 1024 * 1024 * 1024,
-    tb: 1024 * 1024 * 1024 * 1024
-  }
-
-  const match = size.toLowerCase().match(/^(\d+)\s*([a-z]+)$/)
-  if (!match) {
-    throw new Error(`Invalid file size format: ${size}`)
-  }
-
-  const [, value, unit] = match
-  const multiplier = units[unit]
-
-  if (!multiplier) {
-    throw new Error(`Invalid file size unit: ${unit}`)
-  }
-
-  return parseInt(value) * multiplier
-}

package/src/main.ts

@@ -1,21 +0,0 @@
-import { core } from './lib/core'
-import { server } from './lib/server'
-import { launchServer } from './lib/server/utils'
-
-/**
- * The main function for the action.
- * @returns {Promise<void>} Resolves when the action is complete.
- */
-export async function run(): Promise<void> {
-  try {
-    //* Daemon process
-    if (process.argv[2] === '--server') {
-      return server()
-    }
-    //* Base process
-    return launchServer()
-  } catch (error) {
-    // Fail the workflow run if an error occurs
-    if (error instanceof Error) core.setFailed(error.message)
-  }
-}

package/src/post.ts

@@ -1,26 +0,0 @@
-import { readFile } from 'fs/promises'
-import { serverLogFile, serverPort } from './lib/constants'
-import { core } from './lib/core'
-
-/**
- * The out script for the action.
- */
-async function post(): Promise<void> {
-  try {
-    //* Kill the server
-    await fetch(`http://localhost:${serverPort}/shutdown`, {
-      method: 'DELETE'
-    })
-
-    //* Read the logs
-    const logs = await readFile(serverLogFile, 'utf-8')
-    //* Print the logs
-    core.info(logs)
-  } catch (error) {
-    // Fail the workflow run if an error occurs
-    if (error instanceof Error) core.setFailed(error.message)
-  }
-}
-
-// Run the out script
-post()

package/tsconfig.json

@@ -1,19 +0,0 @@
-{
-  "$schema": "https://json.schemastore.org/tsconfig",
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "ESNext",
-    "rootDir": "./src",
-    "moduleResolution": "bundler",
-    "baseUrl": "./",
-    "sourceMap": true,
-    "outDir": "./dist",
-    "noImplicitAny": true,
-    "esModuleInterop": true,
-    "forceConsistentCasingInFileNames": true,
-    "strict": true,
-    "skipLibCheck": true,
-    "newLine": "lf"
-  },
-  "exclude": ["./dist", "./node_modules", "./__tests__", "./coverage"]
-}

package/turbo.json

@@ -1,9 +0,0 @@
-{
-  "$schema": "https://turbo.build/schema.json",
-  "tasks": {
-    "test-build": {
-      "inputs": ["src"],
-      "outputs": ["dist"]
-    }
-  }
-}