git-truck 0.5.0

package/app/parser/src/model.ts

@@ -0,0 +1,77 @@
+export type GitObject = GitBlobObject | GitTreeObject
+
+export interface GitBaseObject {
+  type: "blob" | "tree" | "commit"
+  hash: string
+}
+
+export interface ParserData {
+  repo: string
+  branch: string
+  commit: HydratedGitCommitObject
+  authorUnions: string[][]
+}
+
+export interface GitBlobObject extends GitBaseObject {
+  type: "blob"
+  name: string
+  path: string
+  content?: string
+}
+
+export type HydratedGitObject = HydratedGitBlobObject | HydratedGitTreeObject
+
+export interface HydratedGitBlobObject extends GitBlobObject {
+  noLines: number
+  authors: Record<string, number>
+  unionedAuthors?: Record<string, number>
+  noCommits: number
+  lastChangeEpoch?: number
+  dominantAuthor?: [string, number]
+  isBinary?: boolean
+}
+
+export interface GitTreeObject extends GitBaseObject {
+  type: "tree"
+  name: string
+  path: string
+  children: (GitTreeObject | GitBlobObject)[]
+}
+
+export interface HydratedGitTreeObject extends Omit<GitTreeObject, "children"> {
+  children: (HydratedGitTreeObject | HydratedGitBlobObject)[]
+}
+
+export interface GitCommitObject extends GitBaseObject {
+  type: "commit"
+  tree: GitTreeObject
+  parent: string
+  parent2: string | null
+  author: PersonWithTime
+  committer: PersonWithTime
+  message: string
+  description: string
+  coauthors: Person[]
+}
+
+export interface HydratedGitCommitObject extends Omit<GitCommitObject, "tree"> {
+  tree: HydratedGitTreeObject
+  minNoCommits: number
+  maxNoCommits: number
+  newestLatestChangeEpoch: number
+  oldestLatestChangeEpoch: number
+}
+
+export type GitCommitObjectLight = Omit<GitCommitObject, "tree"> & {
+  tree: string
+}
+
+export interface Person {
+  name: string
+  email: string
+}
+
+export type PersonWithTime = Person & {
+  timestamp: number
+  timezone: string
+}

package/app/parser/src/parse.server.ts

@@ -0,0 +1,276 @@
+import fsSync, { promises as fs, readFileSync } from "fs"
+import {
+  GitBlobObject,
+  GitCommitObject,
+  GitCommitObjectLight,
+  GitTreeObject,
+  Person,
+} from "./model"
+import { log, setLogLevel } from "./log.server"
+import {
+  describeAsyncJob,
+  formatMs,
+  writeRepoToFile,
+  getCurrentBranch,
+  getRepoName,
+  deflateGitObject,
+} from "./util"
+import { emptyGitCommitHash } from "./constants"
+import { resolve , isAbsolute, join} from "path"
+import TruckIgnore from "./TruckIgnore.server"
+import { performance } from "perf_hooks"
+import yargsParser from "yargs-parser"
+import { hydrateData } from "./hydrate.server"
+
+import { } from "@remix-run/node"
+
+export async function findBranchHead(repo: string, branch: string | null) {
+  if (branch === null) branch = await getCurrentBranch(repo)
+
+  const gitFolder = join(repo, ".git")
+  if (!fsSync.existsSync(gitFolder)) {
+    throw Error(`${repo} is not a git repository`)
+  }
+  // Find file containing the branch head
+  const branchPath = join(gitFolder, "refs/heads/" + branch)
+  const absolutePath = join(process.cwd(), branchPath)
+  log.debug("Looking for branch head at " + absolutePath)
+
+  const branchHead = (await fs.readFile(branchPath, "utf-8")).trim()
+  log.debug(`${branch} -> [commit]${branchHead}`)
+
+  return [branchHead, branch]
+}
+
+export async function parseCommitLight(
+  repo: string,
+  hash: string
+): Promise<GitCommitObjectLight> {
+  const rawContent = await deflateGitObject(repo, hash)
+  const commitRegex =
+    /tree (?<tree>.*)\n(?:parent (?<parent>.*)\n)?(?:parent (?<parent2>.*)\n)?author (?<authorName>.*) <(?<authorEmail>.*)> (?<authorTimeStamp>\d*) (?<authorTimeZone>.*)\ncommitter (?<committerName>.*) <(?<committerEmail>.*)> (?<committerTimeStamp>\d*) (?<committerTimeZone>.*)\n(?:gpgsig (?:.|\n)*-----END PGP SIGNATURE-----)?\s*(?<message>.*)\s*(?<description>(?:.|\s)*)/gm
+
+  const match = commitRegex.exec(rawContent)
+  const groups = match?.groups ?? {}
+
+  const tree = groups["tree"]
+  const parent = groups["parent"] ?? emptyGitCommitHash
+  const parent2 = groups["parent2"] ?? null
+  const author = {
+    name: groups["authorName"],
+    email: groups["authorEmail"],
+    timestamp: Number(groups["authorTimeStamp"]),
+    timezone: groups["authorTimeZone"],
+  }
+  const committer = {
+    name: groups["committerName"],
+    email: groups["committerEmail"],
+    timestamp: Number(groups["committerTimeStamp"]),
+    timezone: groups["committerTimeZone"],
+  }
+  const message = groups["message"]
+  const description = groups["description"]
+  const coauthors = description ? getCoAuthors(description) : []
+
+  return {
+    type: "commit",
+    hash,
+    tree,
+    parent,
+    parent2,
+    author,
+    committer,
+    message,
+    description,
+    coauthors,
+  }
+}
+
+export async function parseCommit(
+  repo: string,
+  repoName: string,
+  hash: string
+): Promise<GitCommitObject> {
+  const { tree, ...commit } = await parseCommitLight(repo, hash)
+  const truckignore = new TruckIgnore(repo)
+  return {
+    ...commit,
+    tree: await parseTree(getRepoName(repo), repo, repoName, tree, truckignore),
+  }
+}
+
+function getCoAuthors(description: string) {
+  const coauthorRegex = /.*Co-authored-by: (?<name>.*) <(?<email>.*)>/gm
+  const coauthormatches = description.matchAll(coauthorRegex)
+  let next = coauthormatches.next()
+  const coauthors: Person[] = []
+
+  while (next.value !== undefined) {
+    coauthors.push({
+      name: next.value.groups["name"].trimEnd(),
+      email: next.value.groups["email"],
+    })
+    next = coauthormatches.next()
+  }
+  return coauthors
+}
+
+async function parseTree(
+  path: string,
+  repo: string,
+  name: string,
+  hash: string,
+  truckignore: TruckIgnore
+): Promise<GitTreeObject> {
+  const rawContent = await deflateGitObject(repo, hash)
+  const entries = rawContent.split("\n").filter((x) => x.trim().length > 0)
+
+  const children: (GitTreeObject | GitBlobObject)[] = []
+  for await (const line of entries) {
+    const catFileRegex = /^.+?\s(?<type>\w+)\s(?<hash>.+?)\s+(?<name>.+?)\s*$/g;
+    const groups = catFileRegex.exec(line)?.groups ?? {}
+
+    const type = groups["type"]
+    const hash = groups["hash"]
+    const name = groups["name"]
+
+    if (!truckignore.isAccepted(name)) continue
+    const newPath = [path, name].join("/")
+    log.debug(`Path: ${newPath}`)
+
+    switch (type) {
+      case "tree":
+        children.push(await parseTree(newPath, repo, name, hash, truckignore))
+        break
+      case "blob":
+        children.push(await parseBlob(newPath, repo, name, hash))
+        break
+      default:
+        throw new Error(` type ${type}`)
+    }
+  }
+
+  return {
+    type: "tree",
+    path: path,
+    name,
+    hash,
+    children,
+  }
+}
+
+async function parseBlob(
+  path: string,
+  repo: string,
+  name: string,
+  hash: string
+): Promise<GitBlobObject> {
+  const content = await deflateGitObject(repo, hash)
+  const blob: GitBlobObject = {
+    type: "blob",
+    hash,
+    path,
+    name,
+    content,
+  }
+  return blob
+}
+
+interface truckConfigResponse {
+  unionedAuthors: string[][]
+}
+
+export async function loadTruckConfig(repoDir: string) {
+  try {
+    const truckConfig = JSON.parse(
+      readFileSync(join(repoDir, "truckconfig.json"), "utf-8")
+    ) as truckConfigResponse
+    return truckConfig.unionedAuthors
+  } catch (e) {
+    log.info("No truckignore found: " + e)
+  }
+  return []
+}
+
+export async function parse(rawArgs: string[]) {
+  const args = yargsParser(rawArgs, {
+    configuration: {
+      "duplicate-arguments-array": false,
+    },
+  })
+
+  if (args.log) {
+    setLogLevel(args.log)
+  }
+
+  if (args.help || args.h) {
+    console.log(`Git Visual
+
+  Usage: ./start.sh <args> or ./dev.sh <args>
+
+  Options:
+    --path <path to git repository> (default: current directory)
+    --branch <branch name> (default: checked out branch)
+    --out <output path for json file> (default: ./app/build/data.json)
+    --help, -h: Show this help message`)
+    process.exit(1)
+  }
+
+  const cwd = process.cwd()
+
+  let repoDir = args.path ?? "."
+  if (!isAbsolute(repoDir))
+    repoDir = resolve(cwd, repoDir)
+
+  const branch = args.branch ?? null
+
+  const start = performance.now()
+  const [branchHead, branchName] = await describeAsyncJob(
+    () => findBranchHead(repoDir, branch),
+    "Finding branch head",
+    "Found branch head",
+    "Error finding branch head"
+  )
+  const repoName = getRepoName(repoDir)
+  const repoTree = await describeAsyncJob(
+    () => parseCommit(repoDir, repoName, branchHead),
+    "Parsing commit tree",
+    "Commit tree parsed",
+    "Error parsing commit tree"
+  )
+  const hydratedRepoTree = await describeAsyncJob(
+    () => hydrateData(repoDir, repoTree),
+    "Hydrating commit tree with authorship data",
+    "Commit tree hydrated",
+    "Error hydrating commit tree"
+  )
+
+  const defaultOutPath = resolve(__dirname, `../.temp/${repoName}_${branchName}.json`)
+  let outPath = resolve(args.out ?? defaultOutPath)
+  if (!isAbsolute(outPath))
+    outPath = resolve(cwd, outPath)
+
+  const authorUnions = await loadTruckConfig(repoDir)
+  const data = {
+    repo: repoName,
+    branch: branchName,
+    commit: hydratedRepoTree,
+    authorUnions: authorUnions,
+  }
+  await describeAsyncJob(
+    () =>
+      writeRepoToFile(outPath, data),
+    "Writing data to file",
+    `Wrote data to ${resolve(outPath)}`,
+    `Error writing data to file ${outPath}`
+  )
+  const stop = performance.now()
+
+  log.raw(`\nDone in ${formatMs(stop - start)}`)
+
+  return data
+}
+
+export const exportForTest = {
+  getCoAuthors,
+}

package/app/parser/src/parse.test.ts

@@ -0,0 +1,32 @@
+import { exportForTest } from "./parse.server"
+jest.mock("./TruckIgnore.server")
+jest.setTimeout(15000)
+
+describe("getCoAuthors", () => {
+  it("Should return none", () => {
+    const actual = exportForTest.getCoAuthors("lorem ipsum\n\nCo-authored-by:")
+    expect(actual.length).toBe(0)
+  })
+
+  it("Should return none when empty input", () => {
+    const actual = exportForTest.getCoAuthors("")
+    expect(actual.length).toBe(0)
+  })
+
+  it("Should return 2 authors", () => {
+    const sampleDescription =
+      "did some stuff\n\nCo-authored-by: Bob Bobby <bob@example.com>\nCo-authored-by: Alice Lmao <alice@example.com>"
+    const expected = [
+      {
+        name: "Bob Bobby",
+        email: "bob@example.com",
+      },
+      {
+        name: "Alice Lmao",
+        email: "alice@example.com",
+      },
+    ]
+    const actual = exportForTest.getCoAuthors(sampleDescription)
+    expect(actual).toStrictEqual(expected)
+  })
+})

package/app/parser/src/queue.ts

@@ -0,0 +1,86 @@
+// Adapted from algs4 Implementation of Queue.java
+// https://algs4.cs.princeton.edu/code/edu/princeton/cs/algs4/Queue.java.html
+// algs4, Robert Sedgewick and Kevin Wayne
+
+// helper linked list class
+class QueueNode<T> {
+  item: T
+  next: QueueNode<T>|null = null
+
+  constructor(item : T) {
+    this.item = item
+  }
+}
+
+export class Queue<T> {
+  first : QueueNode<T>|null  // beginning of queue
+  last : QueueNode<T>|null   // end of queue
+  n : number                 // number of elements on queue
+
+  /**
+   * Initializes an empty queue.
+   */
+  constructor() {
+    this.first = null
+    this.last  = null
+    this.n = 0
+  }
+
+  /**
+   * Returns true if this queue is empty.
+   *
+   * @return {@code true} if this queue is empty {@code false} otherwise
+   */
+  isEmpty() : boolean {
+      return this.first == null
+  }
+
+  /**
+   * Returns the number of items in this queue.
+   *
+   * @return the number of items in this queue
+   */
+  size() : number {
+      return this.n
+  }
+
+  /**
+   * Returns the item least recently added to this queue.
+   *
+   * @return the item least recently added to this queue
+   * @throws NoSuchElementException if this queue is empty
+   */
+  peek() : T|null {
+      if (this.isEmpty()) throw new Error("Queue underflow")
+      return this.first!.item // this.first should not be null
+  }
+
+  /**
+   * Adds the item to this queue.
+   *
+   * @param  item the item to add
+   */
+  enqueue(item: T) {
+      const oldlast = this.last
+      this.last = new QueueNode<T>(item)
+      this.last.next = null
+      if (this.isEmpty()) this.first = this.last
+      else           oldlast!.next = this.last
+      this.n++
+  }
+
+  /**
+   * Removes and returns the item on this queue that was least recently added.
+   *
+   * @return the item on this queue that was least recently added
+   * @throws NoSuchElementException if this queue is empty
+   */
+  dequeue() : T {
+      if (this.isEmpty()) throw new Error("Queue underflow")
+      const item = this.first!.item
+      this.first = this.first!.next
+      this.n--
+      if (this.isEmpty()) this.last = null   // to avoid loitering
+      return item
+  }
+}

package/app/parser/src/util.test.ts

@@ -0,0 +1,8 @@
+import { last } from "./util"
+
+describe("last", () => {
+  it("gets the last element of the array", () => {
+    const arr = [1, 2, 3, 4, 5]
+    expect(last(arr)).toBe(5)
+  })
+})

package/app/parser/src/util.ts

@@ -0,0 +1,216 @@
+import { spawn } from "child_process"
+import { existsSync, promises as fs } from "fs"
+import { createSpinner, Spinner } from "nanospinner"
+import { dirname, resolve, sep } from "path"
+import { getLogLevel, log, LOG_LEVEL } from "./log.server"
+import { GitBlobObject, GitTreeObject, ParserData } from "./model"
+import { performance } from "perf_hooks"
+
+export function last<T>(array: T[]) {
+  return array[array.length - 1]
+}
+
+function runProcess(dir: string, command: string, args: string[]) {
+  return new Promise((resolve, reject) => {
+    try {
+      const prcs = spawn(command, args, {
+        cwd: dir,
+      })
+      const chunks: Uint8Array[] = []
+      prcs.stderr.once("data", (buf) => reject(buf.toString().trim()))
+      prcs.stdout.on("data", (buf) => chunks.push(buf))
+      prcs.stdout.on("end", () => {
+        resolve(Buffer.concat(chunks).toString().trim())
+      })
+    } catch (e) {
+      reject(e)
+    }
+  })
+}
+
+export async function gitDiffNumStatParsed(
+  repo: string,
+  a: string,
+  b: string,
+  renamedFiles: Map<string, string>
+) {
+  const diff = await gitDiffNumStat(repo, a, b)
+  const entries = diff.split("\n")
+  const stuff = entries
+    .filter((x) => x.trim().length > 0)
+    .map((x) => x.split(/\t+/))
+    .map(([neg, pos, file]) => {
+      let filePath = file
+      const hasBeenMoved = file.includes("=>")
+      if (hasBeenMoved) {
+        filePath = parseRenamedFile(filePath, renamedFiles)
+      }
+
+      const newestPath = findNewestVersion(filePath, renamedFiles)
+
+      return {
+        neg: parseInt(neg),
+        pos: parseInt(pos),
+        file: newestPath,
+      }
+    })
+  return stuff
+}
+
+function parseRenamedFile(file: string, renamedFiles: Map<string, string>) {
+  const movedFileRegex =
+    /(?:.*{(?<oldPath>.*)\s=>\s(?<newPath>.*)}.*)|(?:^(?<oldPath2>.*) => (?<newPath2>.*))$/gm
+  const replaceRegex = /{.*}/gm
+  const match = movedFileRegex.exec(file)
+  const groups = match?.groups ?? {}
+
+  let oldPath: string
+  let newPath: string
+
+  if (groups["oldPath"] || groups["newPath"]) {
+    const oldP = groups["oldPath"] ?? ""
+    const newP = groups["newPath"] ?? ""
+    oldPath = file.replace(replaceRegex, oldP).replace("//", "/")
+    newPath = file.replace(replaceRegex, newP).replace("//", "/")
+  } else {
+    oldPath = groups["oldPath2"] ?? ""
+    newPath = groups["newPath2"] ?? ""
+  }
+
+  renamedFiles.set(oldPath, newPath)
+  return newPath
+}
+
+function findNewestVersion(path: string, renamedFiles: Map<string, string>) {
+  let newestPath = path
+  let next = renamedFiles.get(newestPath)
+  while (next) {
+    newestPath = next
+    next = renamedFiles.get(newestPath)
+  }
+  return newestPath
+}
+
+export async function lookupFileInTree(
+  tree: GitTreeObject,
+  path: string
+): Promise<GitBlobObject | undefined> {
+  const dirs = path.split("/")
+
+  if (dirs.length < 2) {
+    // We have reached the end of the tree, look for the blob
+    const [file] = dirs
+    const result = tree.children.find(
+      (x) => x.name === file && x.type === "blob"
+    )
+    if (!result) return
+    if (result.type === "tree") return undefined
+    return result
+  }
+  const subtree = tree.children.find((x) => x.name === dirs[0])
+  if (!subtree || subtree.type === "blob") return
+  return await lookupFileInTree(subtree, dirs.slice(1).join("/"))
+}
+
+export async function gitDiffNumStat(repoDir: string, a: string, b: string) {
+  const result = await runProcess(repoDir, "git", ["diff", "--numstat", a, b])
+  return result as string
+}
+
+export async function deflateGitObject(repo: string, hash: string) {
+  const result = await runProcess(repo, "git", ["cat-file", "-p", hash])
+  return result as string
+}
+
+export async function writeRepoToFile(outPath: string, parsedData: ParserData) {
+  const data = JSON.stringify(parsedData, null, 2)
+  const dir = dirname(outPath)
+  if (!existsSync(dir)) {
+    await fs.mkdir(dir, { recursive: true })
+  }
+  await fs.writeFile(outPath, data)
+  return outPath
+}
+
+export function getRepoName(repoDir: string) {
+  return resolve(repoDir).split(sep).slice().reverse()[0]
+}
+
+export async function getCurrentBranch(dir: string) {
+  const result = (await runProcess(dir, "git", [
+    "rev-parse",
+    "--abbrev-ref",
+    "HEAD",
+  ])) as string
+  return result.trim()
+}
+
+export const formatMs = (ms: number) => {
+  if (ms < 1000) {
+    return `${Math.round(ms)}ms`
+  } else {
+    return `${(ms / 1000).toFixed(2)}s`
+  }
+}
+
+export function generateTruckFrames(length: number) {
+  const frames = []
+  for (let i = 0; i < length; i++) {
+    const prefix = " ".repeat(length - i - 1)
+    const frame = `${prefix}🚛\n`
+    frames.push(frame)
+  }
+  return frames
+}
+
+export function createTruckSpinner() {
+  return getLogLevel() === null
+    ? createSpinner("", {
+        interval: 1000 / 20,
+        frames: generateTruckFrames(20),
+      })
+    : null
+}
+
+let spinner: null | Spinner = null
+
+export async function describeAsyncJob<T>(
+  job: () => Promise<T>,
+  beforeMsg: string,
+  afterMsg: string,
+  errorMsg: string
+) {
+  spinner = createTruckSpinner()
+  const success = (text: string, final = false) => {
+    if (getLogLevel() === LOG_LEVEL.SILENT) return
+    if (spinner === null) return log.info(text)
+    spinner.success({ text })
+    if (!final) spinner.start()
+  }
+  const output = (text: string) => {
+    if (spinner) {
+      spinner.update({
+        text,
+        frames: generateTruckFrames(text.length),
+      })
+      spinner.start()
+    } else log.info(text)
+  }
+
+  const error = (text: string) =>
+    spinner === null ? log.error(text) : spinner.error({ text })
+
+  output(beforeMsg)
+  try {
+    const startTime = performance.now()
+    const result = await job()
+    const stopTime = performance.now()
+    const suffix = `[${formatMs(stopTime - startTime)}]`
+    success(`${afterMsg} ${suffix}`, true)
+    return result
+  } catch (e) {
+    error(errorMsg)
+    log.error(e as Error)
+    process.exit(1)
+  }
+}