fire-marshal-ebay 0.0.1-security.2 → 1.0.0

package/confused/.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '15 8 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

package/confused/.github/workflows/golangci-lint.yml

@@ -0,0 +1,28 @@
+name: golangci-lint
+on:
+  push:
+    tags:
+      - v*
+    branches:
+      - main
+  pull_request:
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
+          version: v1.29
+
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+
+          # Optional: golangci-lint command line arguments.
+          # args: --issues-exit-code=0
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true

package/confused/.goreleaser.yml

@@ -0,0 +1,40 @@
+builds:
+  - id: confused
+    binary: confused
+    flags:
+      - -trimpath
+    env:
+      - CGO_ENABLED=0
+    asmflags:
+      - all=-trimpath={{.Env.GOPATH}}
+    gcflags:
+      - all=-trimpath={{.Env.GOPATH}}
+    ldflags: |
+      -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -extldflags '-static'
+    goos:
+      - linux
+      - windows
+      - freebsd
+      - openbsd
+      - darwin
+    goarch:
+      - amd64
+      - 386
+      - arm
+      - arm64
+    ignore:
+      - goos: freebsd
+        goarch: arm64
+
+archives:
+  - id: tgz
+    format: tar.gz
+    replacements:
+        darwin: macOS
+    format_overrides:
+        - goos: windows
+          format: zip
+
+signs:
+  - artifacts: checksum
+

package/confused/CHANGELOG.md

@@ -0,0 +1,31 @@
+# Changelog
+
+- main
+    - New
+    - Changed
+
+- v0.4
+    - New
+        - npm: In case package was found, also check if all the package versions have been unpublished. This makes the package vulnerable to takeover
+        - npm: Check for http & https and GitHub version references
+        - MVN (Maven) support
+    - Changed
+        - Fixed a bug where the pip requirements.txt parser processes a 'tilde equals' sign.
+        - Fixed an issue that would detect git repository urls as matches
+
+- v0.3
+    - New
+        - PHP (composer) support
+        - Command line parameter to let the user to flag namespaces as known-safe
+    - Changed
+        - Python (pypi) dependency definition files that use line continuation are now parsed correctly
+        - Revised the output to clarify the usage
+        - Fixed npm package.json file parsing issues when the source file is not following the specification
+
+- v0.2
+    - Changed
+        - npm registry checkup url
+        - Throttle the rate of requests in case of 429 (Too many requests) responses
+
+- v0.1
+   - Initial release

package/confused/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Visma Security, Joona Hoikkala
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/confused/README.md

@@ -0,0 +1,93 @@
+# Confused
+
+A tool for checking for lingering free namespaces for private package names referenced in dependency configuration
+for Python (pypi) `requirements.txt`, JavaScript (npm) `package.json`, PHP (composer) `composer.json` or MVN (maven) `pom.xml`.
+
+## What is this all about?
+
+On 9th of February 2021, a security researcher Alex Birsan [published an article](https://medium.com/@alex.birsan/dependency-confusion-4a5d60fec610)
+that touched different resolve order flaws in dependency management tools present in multiple programming language ecosystems.
+
+Microsoft [released a whitepaper](https://azure.microsoft.com/en-gb/resources/3-ways-to-mitigate-risk-using-private-package-feeds/)
+describing ways to mitigate the impact, while the root cause still remains.
+
+## Interpreting the tool output
+
+`confused` simply reads through a dependency definition file of an application and checks the public package repositories
+for each dependency entry in that file. It will proceed to report all the package names that are not found in the public
+repositories - a state that implies that a package might be vulnerable to this kind of attack, while this vector has not
+yet been exploited.
+
+This however doesn't mean that an application isn't already being actively exploited. If you know your software is using
+private package repositories, you should ensure that the namespaces for your private packages have been claimed by a
+trusted party (typically yourself or your company).
+
+### Known false positives
+
+Some packaging ecosystems like npm have a concept called "scopes" that can be either private or public. In short it means
+a namespace that has an upper level - the scope. The scopes are not inherently visible publicly, which means that `confused`
+cannot reliably detect if it has been claimed. If your application uses scoped package names, you should ensure that a
+trusted party has claimed the scope name in the public repositories.
+
+## Installation
+
+- [Download](https://github.com/visma-prodsec/confused/releases/latest) a prebuilt binary from [releases page](https://github.com/visma-prodsec/confused/releases/latest), unpack and run!
+
+  _or_
+- If you have recent go compiler installed: `go get -u github.com/visma-prodsec/confused` (the same command works for updating)
+
+  _or_
+- git clone https://github.com/visma-prodsec/confused ; cd confused ; go get ; go build
+
+## Usage
+```
+Usage:
+ ./confused [-l LANGUAGENAME] depfilename.ext
+
+Usage of ./confused:
+  -l string
+        Package repository system. Possible values: "pip", "npm", "composer", "mvn" (default "npm")
+  -s string
+        Comma-separated list of known-secure namespaces. Supports wildcards
+  -v    Verbose output
+
+```
+
+## Example
+
+### Python (PyPI)
+```
+./confused -l pip requirements.txt
+
+Issues found, the following packages are not available in public package repositories:
+ [!] internal_package1
+
+```
+
+### JavaScript (npm)
+```
+./confused -l npm package.json
+
+Issues found, the following packages are not available in public package repositories:
+ [!] internal_package1
+ [!] @mycompany/internal_package1
+ [!] @mycompany/internal_package2
+
+# Example when @mycompany private scope has been registered in npm, using -s
+./confused -l npm -s '@mycompany/*' package.json
+
+Issues found, the following packages are not available in public package repositories:
+ [!] internal_package1
+```
+
+
+### Maven (mvn)
+```
+./confused -l mvn pom.xml
+
+Issues found, the following packages are not available in public package repositories:
+ [!] internal
+ [!] internal/package1
+ [!] internal/_package2
+
+```

package/confused/composer.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"time"
+)
+
+type ComposerJSON struct {
+	Require map[string]string `json:"require"`
+	RequireDev map[string]string `json:"require-dev"`
+}
+
+type ComposerLookup struct {
+	Packages []string
+	Verbose bool
+}
+
+func NewComposerLookup(verbose bool) PackageResolver {
+	return &ComposerLookup{Packages: []string{}, Verbose: verbose}
+}
+
+func (c *ComposerLookup) ReadPackagesFromFile(filename string) error {
+	rawfile, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return err
+	}
+
+	data := ComposerJSON{}
+	err = json.Unmarshal([]byte(rawfile), &data)
+	if err != nil {
+		return err
+	}
+
+	for pkgname := range data.Require {
+		c.Packages = append(c.Packages, pkgname)
+	}
+
+	for pkgname := range data.RequireDev {
+		c.Packages = append(c.Packages, pkgname)
+	}
+
+	return nil
+}
+
+func (c *ComposerLookup) PackagesNotInPublic() []string {
+	notavail := []string{}
+	for _, pkg := range c.Packages {
+		if pkg == "php" {
+			continue
+		}
+
+		if !c.isAvailableInPublic(pkg, 0) {
+			notavail = append(notavail, pkg)
+		}
+	}
+
+	return notavail
+}
+
+func (c *ComposerLookup) isAvailableInPublic(pkgname string, retry int) bool {
+	if retry > 3 {
+		fmt.Printf(" [W] Maximum number of retries exhausted for package %s\n", pkgname)
+
+		return false
+	}
+
+	if c.Verbose {
+		fmt.Printf("Checking: https://packagist.org/packages/%s : ", pkgname)
+	}
+
+	client := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	resp, err := client.Get("https://packagist.org/packages/" + pkgname)
+	if err != nil {
+		fmt.Printf(" [W] Error when trying to request https://packagist.org/packages/%s : %s\n", pkgname, err)
+
+		return false
+	}
+	defer resp.Body.Close()
+
+	if c.Verbose {
+		fmt.Printf("%s\n", resp.Status)
+	}
+
+	if resp.StatusCode == http.StatusOK {
+		return true
+	}
+
+	if resp.StatusCode == 429 {
+		fmt.Printf(" [!] Server responded with 429 (Too many requests), throttling and retrying..\n")
+		time.Sleep(10 * time.Second)
+		retry = retry + 1
+
+		c.isAvailableInPublic(pkgname, retry)
+	}
+
+	return false
+}

package/confused/interfaces.go

@@ -0,0 +1,11 @@
+package main
+
+// A PackageResolver resolves package information from a file
+//
+// ReadPackagesFromFile should take a filepath as input and parse relevant package information into a struct and then return any errors encountered while reading or unmarshalling the file.
+//
+// PackagesNotInPublic should determine whether or not a package is not available in a public package repository and return a slice of all packages not available in a public package repository.
+type PackageResolver interface {
+	ReadPackagesFromFile(string) error
+	PackagesNotInPublic() []string
+}

package/confused/main.go

@@ -0,0 +1,104 @@
+/*
+Package main implements an automated Dependency Confusion scanner.
+
+Original research provided by Alex Birsan.
+
+Original blog post detailing Dependency Confusion : https://medium.com/@alex.birsan/dependency-confusion-4a5d60fec610 .
+*/
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func main() {
+	var resolver PackageResolver
+	lang := ""
+	verbose := false
+	filename := ""
+	safespaces := ""
+	flag.StringVar(&lang, "l", "npm", "Package repository system. Possible values: \"pip\", \"npm\", \"composer\", \"mvn\"")
+	flag.StringVar(&safespaces, "s", "", "Comma-separated list of known-secure namespaces. Supports wildcards")
+	flag.BoolVar(&verbose, "v", false, "Verbose output")
+	flag.Parse()
+
+	// Check that we have a filename
+	if flag.NArg() == 0 {
+		Help()
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	filename = flag.Args()[0]
+	if lang == "pip" {
+		resolver = NewPythonLookup(verbose)
+	} else if lang == "npm" {
+		resolver = NewNPMLookup(verbose)
+	} else if lang == "composer" {
+		resolver = NewComposerLookup(verbose)
+	} else if lang == "mvn" {
+		resolver = NewMVNLookup(verbose)
+	} else {
+		fmt.Printf("Unknown package repository system: %s\n", lang)
+		os.Exit(1)
+	}
+	err := resolver.ReadPackagesFromFile(filename)
+	if err != nil {
+		fmt.Printf("Encountered an error while trying to read packages from file: %s\n", err)
+		os.Exit(1)
+	}
+	outputPackages := removeSafe(resolver.PackagesNotInPublic(), safespaces)
+	PrintResult(outputPackages)
+}
+
+// Help outputs tool usage and help
+func Help() {
+	fmt.Printf("Usage:\n %s [-l LANGUAGENAME] depfilename.ext\n", os.Args[0])
+}
+
+// PrintResult outputs the result of the scanner
+func PrintResult(notavail []string) {
+	if len(notavail) == 0 {
+		fmt.Printf("[*] All packages seem to be available in the public repositories. \n\n" +
+			"In case your application uses private repositories please make sure that those namespaces in \n" +
+			"public repositories are controlled by a trusted party.\n\n")
+		return
+	}
+	fmt.Printf("Issues found, the following packages are not available in public package repositories:\n")
+	for _, n := range notavail {
+		fmt.Printf(" [!] %s\n", n)
+	}
+	os.Exit(1)
+}
+
+// removeSafe removes known-safe package names from the slice
+func removeSafe(packages []string, safespaces string) []string {
+	retSlice := []string{}
+	safeNamespaces := []string{}
+	var ignored bool
+	safeTmp := strings.Split(safespaces, ",")
+	for _, s := range safeTmp {
+		safeNamespaces = append(safeNamespaces, strings.TrimSpace(s))
+	}
+	for _, p := range packages {
+		ignored = false
+		for _, s := range safeNamespaces {
+			ok, err := filepath.Match(s, p)
+			if err != nil {
+				fmt.Printf(" [W] Encountered an error while trying to match a known-safe namespace %s : %s\n", s, err)
+				continue
+			}
+			if ok {
+				ignored = true
+			}
+		}
+		if !ignored {
+			retSlice = append(retSlice, p)
+		}
+	}
+	return retSlice
+}

package/confused/mvn.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// NPMLookup represents a collection of npm packages to be tested for dependency confusion.
+type MVNLookup struct {
+	Packages []MVNPackage
+	Verbose  bool
+}
+
+type MVNPackage struct {
+	Group string
+	Artifact string
+	Version string
+}
+
+// NewNPMLookup constructs an `MVNLookup` struct and returns it.
+func NewMVNLookup(verbose bool) PackageResolver {
+	return &MVNLookup{Packages: []MVNPackage{}, Verbose: verbose}
+}
+
+// ReadPackagesFromFile reads package information from an npm package.json file
+//
+// Returns any errors encountered
+func (n *MVNLookup) ReadPackagesFromFile(filename string) error {
+	rawfile, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return err
+	}
+
+	fmt.Print("Checking: filename: " + filename + "\n")
+
+	var project MavenProject
+	if err := xml.Unmarshal([]byte(rawfile), &project); err != nil {
+		log.Fatalf("unable to unmarshal pom file. Reason: %s\n", err)
+	}
+
+	for _, dep := range project.Dependencies {
+		n.Packages = append(n.Packages, MVNPackage{dep.GroupId, dep.ArtifactId, dep.Version})
+	}
+
+	for _, dep := range project.Build.Plugins {
+		n.Packages = append(n.Packages, MVNPackage{dep.GroupId, dep.ArtifactId, dep.Version})
+	}
+
+	for _, build := range project.Profiles {
+		for _, dep := range build.Build.Plugins {
+			n.Packages = append(n.Packages, MVNPackage{dep.GroupId, dep.ArtifactId, dep.Version})
+		}
+	}
+
+	return nil
+}
+
+// PackagesNotInPublic determines if an npm package does not exist in the public npm package repository.
+//
+// Returns a slice of strings with any npm packages not in the public npm package repository
+func (n *MVNLookup) PackagesNotInPublic() []string {
+	notavail := []string{}
+	for _, pkg := range n.Packages {
+		if !n.isAvailableInPublic(pkg, 0) {
+			notavail = append(notavail, pkg.Group + "/" + pkg.Artifact)
+		}
+	}
+	return notavail
+}
+
+// isAvailableInPublic determines if an npm package exists in the public npm package repository.
+//
+// Returns true if the package exists in the public npm package repository.
+func (n *MVNLookup) isAvailableInPublic(pkg MVNPackage, retry int) bool {
+	if retry > 3 {
+		fmt.Printf(" [W] Maximum number of retries exhausted for package: %s\n", pkg.Group)
+		return false
+	}
+	if pkg.Group == "" {
+		return true
+	}
+
+	group := strings.Replace(pkg.Group, ".", "/",-1)
+	if n.Verbose {
+		fmt.Print("Checking: https://repo1.maven.org/maven2/"+group+"/ ")
+	}
+	resp, err := http.Get("https://repo1.maven.org/maven2/"+group+"/")
+	if err != nil {
+		fmt.Printf(" [W] Error when trying to request https://repo1.maven.org/maven2/"+group+"/ : %s\n", err)
+		return false
+	}
+	defer resp.Body.Close()
+	if n.Verbose {
+		fmt.Printf("%s\n", resp.Status)
+	}
+	if resp.StatusCode == http.StatusOK {
+		npmResp := NpmResponse{}
+		body, _ := ioutil.ReadAll(resp.Body)
+		_ = json.Unmarshal(body, &npmResp)
+		if npmResp.NotAvailable() {
+			if n.Verbose {
+				fmt.Printf("[W] Package %s was found, but all its versions are unpublished, making anyone able to takeover the namespace.\n", pkg.Group)
+			}
+			return false
+		}
+		return true
+	} else if resp.StatusCode == 429 {
+		fmt.Printf(" [!] Server responded with 429 (Too many requests), throttling and retrying...\n")
+		time.Sleep(10 * time.Second)
+		retry = retry + 1
+		n.isAvailableInPublic(pkg, retry)
+	}
+	return false
+}