datadog-sdk-testing 0.4.9 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 12fb89df2cbbf6335c470b7b02af1a7548682c2e
-  data.tar.gz: f72745ecd4294db59b42f7ba4fc223238c08399a
+  metadata.gz: e44e65983bc61091e09056f2a4c0915ef278bd26
+  data.tar.gz: ac179c9a1273e47e0224c21caa1de6762cd431fa
 SHA512:
-  metadata.gz: 50afee3a43b62656b0a681913ca997bdfc22182d3e3a15596d7bffd2b3a9cca7f88269ddc051c47d1c5e5bdfcbcffe8412a4cc5ea6666d8145484935cdcb6205
-  data.tar.gz: eac46f3572c5a28fa6f15d6fa0f3934ac3a5eef22222839487a1d465b06d0fb63cf53e5501da0ad005e1f0e541051e08f9d2603f158d533d284ce82d98675931
+  metadata.gz: d763f040338c25d0db5ad817055906dfff23f2103428b6ae322ef4b9b43829acadbab321fc29ef7dc3cb8f0a7494e6c171ee0efa9edf34795da45b7d83f13402
+  data.tar.gz: aacc3d99aeb93c3badb7564f10afbf47cd9352c7eb73fb2bdfa8d3541a30ce11c60c61fd1d5eeda1da8c86ba0d8bdba4ea96ce4bdf10540d91b32852d159ffd3

data/README.md

@@ -30,3 +30,9 @@ ENV['SDK_HOME'] = File.dirname(__FILE__)
 spec = Gem::Specification.find_by_name 'datadog-sdk-testing'
 load "#{spec.gem_dir}/lib/tasks/sdk.rake"
 ```
+
+if you wish to enable the remote facilities for the requirement conflict checking [tool](https://github.com/DataDog/datadog-sdk-testing/blob/master/lib/tasks/ci/hooks/pre-commit.py) you will have to install [github3.py](https://github.com/sigmavirus24/github3.p://github.com/sigmavirus24/github3.py) (pre-release):
+```
+pip install --pre github3.py
+```
+By default this tool is designed to be used as a pre-commit hook and runs only on local repos. Due to the duration of the validation runs it's not recommended to enable remote repos when using as a pre-commit hook. We perform requirement validation in our continuous integration environment, so it's definitely not mandatory.

data/lib/config/CHANGELOG.md

@@ -0,0 +1,8 @@
+# CHANGELOG - Skeleton
+
+0.1.0/ Unreleased
+==================
+
+### Changes
+
+* [FEATURE] adds skeleton integration.

data/lib/tasks/ci/common.rb

@@ -175,6 +175,7 @@ def generate_skeleton(integration)
   copy_skeleton('lib/config/metadata.csv', 'metadata.csv', integration)
   copy_skeleton('lib/config/requirements.txt', 'requirements.txt', integration)
   copy_skeleton('lib/config/README.md', 'README.md', integration)
+  copy_skeleton('lib/config/CHANGELOG.md', 'CHANGELOG.md', integration)
   copy_skeleton('lib/config/conf.yaml.example', 'conf.yaml.example', integration)
 end
 
@@ -260,6 +261,28 @@ class Wait
   end
 end
 
+def travis_pr?
+  !ENV['TRAVIS'].nil? && ENV['TRAVIS_EVENT_TYPE'] == 'pull_request'
+end
+
+def can_skip?
+  return false, [] unless travis_pr?
+
+  modified_checks = []
+  puts "Comparing #{ENV['TRAVIS_PULL_REQUEST_SHA']} with #{ENV['TRAVIS_BRANCH']}"
+  git_output = `git diff --name-only #{ENV['TRAVIS_BRANCH']}...#{ENV['TRAVIS_PULL_REQUEST_SHA']}`
+  puts "Git diff: \n#{git_output}"
+  git_output.each_line do |filename|
+    filename.strip!
+    puts filename
+    return false, [] if filename.split('/').length < 2
+
+    check_name = filename.split('/')[0]
+    modified_checks << check_name unless modified_checks.include? check_name
+  end
+  [true, modified_checks]
+end
+
 namespace :ci do
   namespace :common do
     task :before_install do |t|
@@ -323,8 +346,8 @@ namespace :ci do
       flavors = attr[:flavor]
       sdkhome = ENV['SDK_HOME'] || Dir.pwd
       filter = ENV['NOSE_FILTER'] || '1'
-      nose_command = in_venv ? 'venv/bin/nosetests' : 'nosetests'
 
+      nose_command = in_venv ? 'venv/bin/nosetests' : 'nosetests'
       nose = if flavors.include?('default')
                "(not requires) and #{filter}"
              else

data/lib/tasks/ci/hooks/pre-commit.py

@@ -4,62 +4,196 @@ import fnmatch
 import os
 import sys
 
-OK = 0
+try:
+    import github3
+except ImportError:
+    github3 = None
+
+
 ERR = 1
 
-def get_files(fname):
-    matches = []
-    for root, dirnames, filenames in os.walk('.'):
-        for filename in fnmatch.filter(filenames, fname):
-            matches.append(os.path.join(root, filename))
+ENV_TOKEN = 'GITHUB_TOKEN'
+ENV_USER = 'GITHUB_USER'
+ENV_PASS = 'GITHUB_PASS'
+
+
+class BadGithubRepo(Exception):
+    pass
+
 
-    return matches
+def two_fa():
+    code = ''
+    while not code:
+        code = raw_input('Enter Github 2FA Code: ')
 
+    return code
 
-def process_requirements(reqs, fname):
+
+class RequirementsAnalyzer(object):
     SPECIFIERS = ['==', '!=' '<=', '>=', '<', '>']
 
-    print "processing... {}".format(fname)
-    with open(fname) as f:
-        content = f.readlines()
+    def __init__(self, remote, local, patterns=['requirements.txt'], verbose=False):
+        self.api = None
+        self.remote_sources = remote
+        self.local_sources = local
+        self.req_patterns = patterns
+        self.verbose = verbose
+
+        if github3:
+            if os.environ.get(ENV_TOKEN):
+                self.api = github3.login(
+                    token=os.environ.get(ENV_TOKEN)
+                )
+            elif os.environ.get(ENV_USER) and os.environ.get(ENV_PASS):
+                self.api = github3.login(
+                    os.environ.get(ENV_USER),
+                    os.environ(ENV_PASS),
+                    two_factor_callback=two_fa
+                )
+
+    def get_repo_requirements(self, repo):
+        reqs = {}
+
+        _repo = repo.split('/')
+        if len(_repo) is not 2:
+            raise BadGithubRepo
 
-    for line in content:
-        line = "".join(line.split())
-        for specifier in SPECIFIERS:
-            idx = line.find(specifier)
-            if idx < 0:
+        org = _repo[0]
+        repository = _repo[1]
+        gh_repo = self.api.repository(org, repository)
+
+        contents = gh_repo.directory_contents('/', return_as=dict)
+        files = {}
+        for entry, content in contents.iteritems():
+            if content.type != "dir":
+                files[content.name] = content
                 continue
 
-            req = line[:idx]
-            specifier = line[idx:]
-
-            if req in reqs and reqs[req][0] != specifier:
-                # version mismatch
-                print "There's a version mismatch with {req} " \
-                    " {spec} and {prev_spec} defined in {src}.".format(
-                        req=req,
-                        spec=specifier,
-                        prev_spec=reqs[req][0],
-                        src=reqs[req][1]
-                    )
-                sys.exit(ERR)
-            elif req not in reqs:
-                reqs[req] = (specifier, fname)
-                break
-
-
-requirements = {}
-files = get_files('requirements.txt')
-
-for f in files:
-    process_requirements(requirements, f)
-
-print "No requirement version conflicts found. Looking good... ;)"
-for req, spec in requirements.iteritems():
-    print "{req}{spec} first found in {fname}".format(
-        req=req,
-        spec=spec[0],
-        fname=spec[1]
-    )
-
-sys.exit(OK)
+            req = gh_repo.file_contents("/{}/requirements.txt".format(entry))
+            reqs[entry] = req.decoded
+
+        fmatches = []
+        for pattern in self.req_patterns:
+            fmatches.extend(fnmatch.filter(files.keys(), pattern))
+
+        for match in fmatches:
+            req = gh_repo.file_contents(files[match].path)
+            reqs[match] = req.decoded
+
+        return reqs
+
+    def get_local_files(self):
+        matches = []
+        for src in self.local_sources:
+            for pattern in self.req_patterns:
+                for root, dirnames, filenames in os.walk(src):
+                    for filename in fnmatch.filter(filenames, pattern):
+                        matches.append(os.path.join(root, filename))
+
+        return matches
+
+    def get_local_contents(self, files):
+        local_reqs = {}
+        for fname in files:
+            with open(fname) as f:
+                content = f.read()
+
+            local_reqs[fname] = content
+
+        return local_reqs
+
+    def get_all_requirements(self):
+        reqs = {}
+        if self.api:
+            for repo in self.remote_sources:
+                try:
+                    requirements = self.get_repo_requirements(repo)
+                    if requirements:
+                        reqs[repo] = requirements
+                except BadGithubRepo:
+                    print 'Unable to get repo requirements for {} - skipping.'.format(repo)
+        else:
+            print 'No Github API set (missing creds?) cant crawl remotes.'
+
+        for local in self.local_sources:
+            requirements = self.get_local_contents(self.get_local_files())
+            reqs[local] = requirements
+
+        return reqs
+
+    def process_requirements(self, sources):
+        err = 0
+        reqs = {}
+
+        for source, requirements in sources.iteritems():
+            for integration, content in requirements.iteritems():
+                if self.verbose:
+                    print "processing... {}/{}".format(source, integration)
+                mycontent = content.splitlines()
+
+                for line in mycontent:
+                    line = "".join(line.split())
+                    for specifier in self.SPECIFIERS:
+                        idx = line.find(specifier)
+                        if idx < 0:
+                            continue
+
+                        req = line[:idx]
+                        specifier = line[idx:]
+
+                        if req in reqs and reqs[req][0] != specifier:
+                            # version mismatch
+                            print "There's a version mismatch with {req} " \
+                                " {spec} and {prev_spec} defined in {src} " \
+                                "@ {repo}.".format(
+                                    req=req,
+                                    spec=specifier,
+                                    prev_spec=reqs[req][0],
+                                    src=reqs[req][1],
+                                    repo=reqs[req][2]
+                                )
+                            err = ERR
+                            break
+                        elif req not in reqs:
+                            reqs[req] = (specifier, integration, source)
+                            break
+
+        return err, reqs
+
+
+def str2bool(v):
+    return v.lower() in ("yes", "true", "t", "1")
+
+def main(args):
+    remote = local = []
+    verbose = str2bool(os.environ.get('VERBOSE', 'false'))
+    if not len(args):
+        remote = [repo.strip() for repo in os.environ.get('REQ_REMOTES', '').split(',')]
+        local = [repo.strip() for repo in os.environ.get('REQ_LOCALS', '').split(',')]
+    elif len(args) == 1:
+        local = [repo.strip() for repo in args[0].split(',')]
+    elif len(args) == 2:
+        local = [repo.strip() for repo in args[0].split(',')]
+        remote = [repo.strip() for repo in args[1].split(',')]
+    else:
+        local = ['.']
+
+    analyzer = RequirementsAnalyzer(
+        remote=remote, local=local, patterns=['requirements*.txt'], verbose=verbose)
+
+    err, reqs = analyzer.process_requirements(analyzer.get_all_requirements())
+    if not err:
+        print "No requirement version conflicts found. Looking good... ;)"
+        if verbose:
+            for requirement, spec in reqs.iteritems():
+                print "{req}{spec} first found in {fname} @ {source}".format(
+                    req=requirement,
+                    spec=spec[0],
+                    fname=spec[1],
+                    source=spec[2]
+                )
+
+    sys.exit(err)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

data/lib/tasks/sdk.rake

@@ -147,8 +147,15 @@ namespace :ci do
     check_env
     puts 'Assuming you are running these tests locally' unless ENV['TRAVIS']
     flavor = args[:flavor] || ENV['TRAVIS_FLAVOR'] || 'default'
+    can_skip, checks = can_skip?
+    can_skip &&= !%w(default).include?(flavor)
+
     flavors = flavor.split(',')
     flavors.each do |f|
+      if can_skip && !checks.include?(flavor)
+        puts "skipping #{flavor} tests, not affected by the change".yellow
+        next
+      end
       Rake::Task["ci:#{f}:execute"].invoke
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datadog-sdk-testing
 version: !ruby/object:Gem::Version
-  version: 0.4.9
+  version: 0.5.0
 platform: ruby
 authors:
 - Jaime Fullaondo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-03 00:00:00.000000000 Z
+date: 2017-02-13 00:00:00.000000000 Z
 dependencies: []
 description: Datadog Integration SDK testing/scaffolding gem
 email: jaime.fullaondo@datadoghq.com
@@ -18,6 +18,7 @@ extra_rdoc_files: []
 files:
 - LICENSE
 - README.md
+- lib/config/CHANGELOG.md
 - lib/config/README.md
 - lib/config/check.py
 - lib/config/ci/skeleton.rake