@point3/node-rdkafka 3.6.0-1

package/deps/librdkafka/packaging/nuget/artifact.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+#
+#
+# Collects CI artifacts from S3 storage, downloading them
+# to a local directory.
+#
+# The artifacts' folder in the S3 bucket must have the following token
+# format:
+#  <token>-[<value>]__   (repeat)
+#
+# Recognized tokens (unrecognized tokens are ignored):
+#  p       - project (e.g., "confluent-kafka-python")
+#  bld     - builder (e.g., "travis")
+#  plat    - platform ("osx", "linux", ..)
+#  arch    - arch ("x64", ..)
+#  tag     - git tag
+#  sha     - git sha
+#  bid     - builder's build-id
+#  bldtype - Release, Debug (appveyor)
+#
+# Example:
+#   p-confluent-kafka-python__bld-travis__plat-linux__tag-__sha-112130ce297656ea1c39e7c94c99286f95133a24__bid-271588764__/confluent_kafka-0.11.0-cp35-cp35m-manylinux1_x86_64.whl
+
+
+import re
+import os
+import boto3
+
+import packaging
+
+s3_bucket = 'librdkafka-ci-packages'
+dry_run = False
+
+
+class Artifact (object):
+    def __init__(self, arts, path, info=None):
+        self.path = path
+        # Remove unexpanded AppVeyor $(..) tokens from filename
+        self.fname = re.sub(r'\$\([^\)]+\)', '', os.path.basename(path))
+        slpath = os.path.join(os.path.dirname(path), self.fname)
+        if os.path.isfile(slpath):
+            # Already points to local file in correct location
+            self.lpath = slpath
+        else:
+            # Prepare download location in dlpath
+            self.lpath = os.path.join(arts.dlpath, slpath)
+
+        if info is None:
+            self.info = dict()
+        else:
+            # Assign the map and convert all keys to lower case
+            self.info = {k.lower(): v for k, v in info.items()}
+            # Rename values, e.g., 'plat':'linux' to 'plat':'debian'
+            for k, v in self.info.items():
+                rdict = packaging.rename_vals.get(k, None)
+                if rdict is not None:
+                    self.info[k] = rdict.get(v, v)
+
+        # Score value for sorting
+        self.score = 0
+
+        # AppVeyor symbol builds are of less value
+        if self.fname.find('.symbols.') != -1:
+            self.score -= 10
+
+        self.arts = arts
+        arts.artifacts.append(self)
+
+    def __repr__(self):
+        return self.path
+
+    def __lt__(self, other):
+        return self.score < other.score
+
+    def download(self):
+        """ Download artifact from S3 and store in local directory .lpath.
+            If the artifact is already downloaded nothing is done. """
+        if os.path.isfile(self.lpath) and os.path.getsize(self.lpath) > 0:
+            return
+        print('Downloading %s -> %s' % (self.path, self.lpath))
+        if dry_run:
+            return
+        ldir = os.path.dirname(self.lpath)
+        if not os.path.isdir(ldir):
+            os.makedirs(ldir, 0o755)
+        self.arts.s3_bucket.download_file(self.path, self.lpath)
+
+
+class Artifacts (object):
+    def __init__(self, match, dlpath):
+        super(Artifacts, self).__init__()
+        self.match = match
+        self.artifacts = list()
+        # Download directory (make sure it ends with a path separator)
+        if not dlpath.endswith(os.path.sep):
+            dlpath = os.path.join(dlpath, '')
+        self.dlpath = dlpath
+        if not os.path.isdir(self.dlpath):
+            if not dry_run:
+                os.makedirs(self.dlpath, 0o755)
+
+    def collect_single(self, path, req_tag=True):
+        """ Collect single artifact, be it in S3 or locally.
+        :param: path string: S3 or local (relative) path
+        :param: req_tag bool: Require tag to match.
+        """
+
+        print('?  %s' % path)
+
+        # For local files, strip download path.
+        # Also ignore any parent directories.
+        if path.startswith(self.dlpath):
+            folder = os.path.basename(os.path.dirname(path[len(self.dlpath):]))
+        else:
+            folder = os.path.basename(os.path.dirname(path))
+
+        # The folder contains the tokens needed to perform
+        # matching of project, gitref, etc.
+        rinfo = re.findall(r'(?P<tag>[^-]+)-(?P<val>.*?)__', folder)
+        if rinfo is None or len(rinfo) == 0:
+            print('Incorrect folder/file name format for %s' % folder)
+            return None
+
+        info = dict(rinfo)
+
+        # Ignore AppVeyor Debug builds
+        if info.get('bldtype', '').lower() == 'debug':
+            print('Ignoring debug artifact %s' % folder)
+            return None
+
+        tag = info.get('tag', None)
+        if tag is not None and (len(tag) == 0 or tag.startswith('$(')):
+            # AppVeyor doesn't substite $(APPVEYOR_REPO_TAG_NAME)
+            # with an empty value when not set, it leaves that token
+            # in the string - so translate that to no tag.
+            del info['tag']
+
+        # Match tag or sha to gitref
+        unmatched = list()
+        for m, v in self.match.items():
+            if m not in info or info[m] != v:
+                unmatched.append(m)
+
+        # Make sure all matches were satisfied, unless this is a
+        # common artifact.
+        if info.get('p', '') != 'common' and len(unmatched) > 0:
+            print(info)
+            print('%s: %s did not match %s' %
+                  (info.get('p', None), folder, unmatched))
+            return None
+
+        return Artifact(self, path, info)
+
+    def collect_s3(self):
+        """ Collect and download build-artifacts from S3 based on
+        git reference """
+        print(
+            'Collecting artifacts matching %s from S3 bucket %s' %
+            (self.match, s3_bucket))
+        self.s3 = boto3.resource('s3')
+        self.s3_bucket = self.s3.Bucket(s3_bucket)
+        self.s3_client = boto3.client('s3')
+        for item in self.s3_client.list_objects(
+                Bucket=s3_bucket, Prefix='librdkafka/').get('Contents'):
+            self.collect_single(item.get('Key'))
+
+        for a in self.artifacts:
+            a.download()
+
+    def collect_local(self, path, req_tag=True):
+        """ Collect artifacts from a local directory possibly previously
+        collected from s3 """
+        for f in [os.path.join(dp, f) for dp, dn,
+                  filenames in os.walk(path) for f in filenames]:
+            if not os.path.isfile(f):
+                continue
+            self.collect_single(f, req_tag)

package/deps/librdkafka/packaging/nuget/cleanup-s3.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+#
+# Clean up test builds from librdkafka's S3 bucket.
+# This also covers python builds.
+
+import re
+from datetime import datetime, timezone
+import boto3
+import argparse
+
+# Collects CI artifacts from S3 storage, downloading them
+# to a local directory, or collecting already downloaded artifacts from
+# local directory.
+#
+# The artifacts' folder in the S3 bucket must have the following token
+# format:
+#  <token>-[<value>]__   (repeat)
+#
+# Recognized tokens (unrecognized tokens are ignored):
+#  p       - project (e.g., "confluent-kafka-python")
+#  bld     - builder (e.g., "travis")
+#  plat    - platform ("osx", "linux", ..)
+#  arch    - arch ("x64", ..)
+#  tag     - git tag
+#  sha     - git sha
+#  bid     - builder's build-id
+#  bldtype - Release, Debug (appveyor)
+#  lnk     - std, static
+#
+# Example:
+#   librdkafka/p-librdkafka__bld-travis__plat-linux__arch-x64__tag-v0.0.62__sha-d051b2c19eb0c118991cd8bc5cf86d8e5e446cde__bid-1562.1/librdkafka.tar.gz
+
+
+s3_bucket = 'librdkafka-ci-packages'
+
+
+def may_delete(path):
+    """ Returns true if S3 object path is eligible for deletion, e.g.
+        has a non-release/rc tag. """
+
+    # The path contains the tokens needed to perform
+    # matching of project, gitref, etc.
+    rinfo = re.findall(r'(?P<tag>[^-]+)-(?P<val>.*?)(?:__|$)', path)
+    if rinfo is None or len(rinfo) == 0:
+        print(f"Incorrect folder/file name format for {path}")
+        return False
+
+    info = dict(rinfo)
+
+    tag = info.get('tag', None)
+    if tag is not None and (len(tag) == 0 or tag.startswith('$(')):
+        # AppVeyor doesn't substite $(APPVEYOR_REPO_TAG_NAME)
+        # with an empty value when not set, it leaves that token
+        # in the string - so translate that to no tag.
+        del info['tag']
+        tag = None
+
+    if tag is None:
+        return True
+
+    if re.match(r'^v?\d+\.\d+\.\d+(-?RC\d+)?$', tag,
+                flags=re.IGNORECASE) is None:
+        return True
+
+    return False
+
+
+def collect_s3(s3, min_age_days=60):
+    """ Collect artifacts from S3 """
+    now = datetime.now(timezone.utc)
+    eligible = []
+    totcnt = 0
+    # note: list_objects will return at most 1000 objects per call,
+    #       use continuation token to read full list.
+    cont_token = None
+    more = True
+    while more:
+        if cont_token is not None:
+            res = s3.list_objects_v2(Bucket=s3_bucket,
+                                     ContinuationToken=cont_token)
+        else:
+            res = s3.list_objects_v2(Bucket=s3_bucket)
+
+        if res.get('IsTruncated') is True:
+            cont_token = res.get('NextContinuationToken')
+        else:
+            more = False
+
+        for item in res.get('Contents'):
+            totcnt += 1
+            age = (now - item.get('LastModified')).days
+            path = item.get('Key')
+            if age >= min_age_days and may_delete(path):
+                eligible.append(path)
+
+    return (eligible, totcnt)
+
+
+def chunk_list(lst, cnt):
+    """ Split list into lists of cnt """
+    for i in range(0, len(lst), cnt):
+        yield lst[i:i + cnt]
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--delete",
+                        help="WARNING! Don't just check, actually delete "
+                        "S3 objects.",
+                        action="store_true")
+    parser.add_argument("--age", help="Minimum object age in days.",
+                        type=int, default=360)
+
+    args = parser.parse_args()
+    dry_run = args.delete is not True
+    min_age_days = args.age
+
+    if dry_run:
+        op = "Eligible for deletion"
+    else:
+        op = "Deleting"
+
+    s3 = boto3.client('s3')
+
+    # Collect eligible artifacts
+    eligible, totcnt = collect_s3(s3, min_age_days=min_age_days)
+    print(f"{len(eligible)}/{totcnt} eligible artifacts to delete")
+
+    # Delete in chunks of 1000 (max what the S3 API can do)
+    for chunk in chunk_list(eligible, 1000):
+        print(op + ":\n" + '\n'.join(chunk))
+        if dry_run:
+            continue
+
+        res = s3.delete_objects(Bucket=s3_bucket,
+                                Delete={
+                                    'Objects': [{'Key': x} for x in chunk],
+                                    'Quiet': True
+                                })
+        errors = res.get('Errors', [])
+        if len(errors) > 0:
+            raise Exception(f"Delete failed: {errors}")

package/deps/librdkafka/packaging/nuget/nuget.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+#
+# Front-end for nuget that runs nuget in a docker image.
+
+set -ex
+
+if [[ -f /.dockerenv ]]; then
+    echo "Inside docker"
+
+    pushd $(dirname $0)
+
+    nuget $*
+
+    popd
+
+else
+    echo "Running docker image"
+    docker run -v $(pwd):/io mono:latest /io/$0 $*
+fi
+

package/deps/librdkafka/packaging/nuget/nugetpackage.py

@@ -0,0 +1,278 @@
+#!/usr/bin/env python3
+#
+# Create NuGet package
+#
+
+import os
+import tempfile
+import shutil
+import subprocess
+from packaging import Package, Mapping
+
+
+class NugetPackage (Package):
+    """ All platforms, archs, et.al, are bundled into one set of
+        NuGet output packages: "main", redist and symbols """
+
+    # See .semamphore/semaphore.yml for where these are built.
+    mappings = [
+        Mapping({'arch': 'x64',
+                 'plat': 'linux',
+                 'lnk': 'std'},
+                'librdkafka.tgz',
+                './usr/local/include/librdkafka/rdkafka.h',
+                'build/native/include/librdkafka/rdkafka.h'),
+        Mapping({'arch': 'x64',
+                 'plat': 'linux',
+                 'lnk': 'std'},
+                'librdkafka.tgz',
+                './usr/local/include/librdkafka/rdkafkacpp.h',
+                'build/native/include/librdkafka/rdkafkacpp.h'),
+        Mapping({'arch': 'x64',
+                 'plat': 'linux',
+                 'lnk': 'std'},
+                'librdkafka.tgz',
+                './usr/local/include/librdkafka/rdkafka_mock.h',
+                'build/native/include/librdkafka/rdkafka_mock.h'),
+
+        Mapping({'arch': 'x64',
+                 'plat': 'linux',
+                 'lnk': 'std'},
+                'librdkafka.tgz',
+                './usr/local/share/doc/librdkafka/README.md',
+                'README.md'),
+        Mapping({'arch': 'x64',
+                 'plat': 'linux',
+                 'lnk': 'std'},
+                'librdkafka.tgz',
+                './usr/local/share/doc/librdkafka/CONFIGURATION.md',
+                'CONFIGURATION.md'),
+        Mapping({'arch': 'x64',
+                 'plat': 'osx',
+                 'lnk': 'all'},
+                'librdkafka.tgz',
+                './usr/local/share/doc/librdkafka/LICENSES.txt',
+                'LICENSES.txt'),
+
+        # OSX x64
+        Mapping({'arch': 'x64',
+                 'plat': 'osx'},
+                'librdkafka.tgz',
+                './usr/local/lib/librdkafka.dylib',
+                'runtimes/osx-x64/native/librdkafka.dylib'),
+        # OSX arm64
+        Mapping({'arch': 'arm64',
+                 'plat': 'osx'},
+                'librdkafka.tgz',
+                './usr/local/lib/librdkafka.1.dylib',
+                'runtimes/osx-arm64/native/librdkafka.dylib'),
+
+        # Linux glibc centos8 x64 with GSSAPI
+        Mapping({'arch': 'x64',
+                 'plat': 'linux',
+                 'dist': 'centos8',
+                 'lnk': 'std'},
+                'librdkafka.tgz',
+                './usr/local/lib/librdkafka.so.1',
+                'runtimes/linux-x64/native/librdkafka.so'),
+        # Linux glibc centos8 x64 without GSSAPI (no external deps)
+        Mapping({'arch': 'x64',
+                 'plat': 'linux',
+                 'dist': 'centos8',
+                 'lnk': 'all'},
+                'librdkafka.tgz',
+                './usr/local/lib/librdkafka.so.1',
+                'runtimes/linux-x64/native/centos8-librdkafka.so'),
+        # Linux glibc centos8 arm64 without GSSAPI (no external deps)
+        Mapping({'arch': 'arm64',
+                 'plat': 'linux',
+                 'dist': 'centos8',
+                 'lnk': 'all'},
+                'librdkafka.tgz',
+                './usr/local/lib/librdkafka.so.1',
+                'runtimes/linux-arm64/native/librdkafka.so'),
+
+        # Linux musl alpine x64 without GSSAPI (no external deps)
+        Mapping({'arch': 'x64',
+                 'plat': 'linux',
+                 'dist': 'alpine',
+                 'lnk': 'all'},
+                'librdkafka.tgz',
+                './usr/local/lib/librdkafka.so.1',
+                'runtimes/linux-x64/native/alpine-librdkafka.so'),
+
+        # Common Win runtime
+        Mapping({'arch': 'x64',
+                 'plat': 'win'},
+                'msvcr140.zip',
+                'vcruntime140.dll',
+                'runtimes/win-x64/native/vcruntime140.dll'),
+        Mapping({'arch': 'x64',
+                 'plat': 'win'},
+                'msvcr140.zip',
+                'msvcp140.dll', 'runtimes/win-x64/native/msvcp140.dll'),
+
+        # matches x64 librdkafka.redist.zip
+        Mapping({'arch': 'x64',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/bin/v142/x64/Release/librdkafka.dll',
+                'runtimes/win-x64/native/librdkafka.dll'),
+        Mapping({'arch': 'x64',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/bin/v142/x64/Release/librdkafkacpp.dll',
+                'runtimes/win-x64/native/librdkafkacpp.dll'),
+        Mapping({'arch': 'x64',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/bin/v142/x64/Release/libcrypto-3-x64.dll',
+                'runtimes/win-x64/native/libcrypto-3-x64.dll'),
+        Mapping({'arch': 'x64',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/bin/v142/x64/Release/libssl-3-x64.dll',
+                'runtimes/win-x64/native/libssl-3-x64.dll'),
+        Mapping({'arch': 'x64',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/bin/v142/x64/Release/zlib1.dll',
+                'runtimes/win-x64/native/zlib1.dll'),
+        Mapping({'arch': 'x64',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/bin/v142/x64/Release/zstd.dll',
+                'runtimes/win-x64/native/zstd.dll'),
+        Mapping({'arch': 'x64',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/bin/v142/x64/Release/libcurl.dll',
+                'runtimes/win-x64/native/libcurl.dll'),
+        # matches x64 librdkafka.redist.zip, lib files
+        Mapping({'arch': 'x64',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/lib/v142/x64/Release/librdkafka.lib',
+                'build/native/lib/win/x64/win-x64-Release/v142/librdkafka.lib'  # noqa: E501
+                ),
+        Mapping({'arch': 'x64',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/lib/v142/x64/Release/librdkafkacpp.lib',
+                'build/native/lib/win/x64/win-x64-Release/v142/librdkafkacpp.lib'  # noqa: E501
+                ),
+
+        Mapping({'arch': 'x86',
+                 'plat': 'win'},
+                'msvcr140.zip',
+                'vcruntime140.dll',
+                'runtimes/win-x86/native/vcruntime140.dll'),
+        Mapping({'arch': 'x86',
+                 'plat': 'win'},
+                'msvcr140.zip',
+                'msvcp140.dll', 'runtimes/win-x86/native/msvcp140.dll'),
+
+        # matches Win32 librdkafka.redist.zip
+        Mapping({'arch': 'x86',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/bin/v142/Win32/Release/librdkafka.dll',
+                'runtimes/win-x86/native/librdkafka.dll'),
+        Mapping({'arch': 'x86',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/bin/v142/Win32/Release/librdkafkacpp.dll',
+                'runtimes/win-x86/native/librdkafkacpp.dll'),
+        Mapping({'arch': 'x86',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/bin/v142/Win32/Release/libcrypto-3.dll',
+                'runtimes/win-x86/native/libcrypto-3.dll'),
+        Mapping({'arch': 'x86',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/bin/v142/Win32/Release/libssl-3.dll',
+                'runtimes/win-x86/native/libssl-3.dll'),
+
+        Mapping({'arch': 'x86',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/bin/v142/Win32/Release/zlib1.dll',
+                'runtimes/win-x86/native/zlib1.dll'),
+        Mapping({'arch': 'x86',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/bin/v142/Win32/Release/zstd.dll',
+                'runtimes/win-x86/native/zstd.dll'),
+        Mapping({'arch': 'x86',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/bin/v142/Win32/Release/libcurl.dll',
+                'runtimes/win-x86/native/libcurl.dll'),
+
+        # matches Win32 librdkafka.redist.zip, lib files
+        Mapping({'arch': 'x86',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/lib/v142/Win32/Release/librdkafka.lib',
+                'build/native/lib/win/x86/win-x86-Release/v142/librdkafka.lib'  # noqa: E501
+                ),
+        Mapping({'arch': 'x86',
+                 'plat': 'win'},
+                'librdkafka.redist*',
+                'build/native/lib/v142/Win32/Release/librdkafkacpp.lib',
+                'build/native/lib/win/x86/win-x86-Release/v142/librdkafkacpp.lib'  # noqa: E501
+                )
+    ]
+
+    def __init__(self, version, arts):
+        if version.startswith('v'):
+            version = version[1:]  # Strip v prefix
+        super(NugetPackage, self).__init__(version, arts)
+
+    def cleanup(self):
+        if os.path.isdir(self.stpath):
+            shutil.rmtree(self.stpath)
+
+    def build(self, buildtype):
+        """ Build single NuGet package for all its artifacts. """
+
+        # NuGet removes the prefixing v from the version.
+        vless_version = self.kv['version']
+        if vless_version[0] == 'v':
+            vless_version = vless_version[1:]
+
+        self.stpath = tempfile.mkdtemp(prefix="out-", suffix="-%s" % buildtype,
+                                       dir=".")
+
+        self.render('librdkafka.redist.nuspec')
+        self.copy_template('librdkafka.redist.targets',
+                           destpath=os.path.join('build', 'native'))
+        self.copy_template('librdkafka.redist.props',
+                           destpath='build')
+
+        # Generate template tokens for artifacts
+        for a in self.arts.artifacts:
+            if 'bldtype' not in a.info:
+                a.info['bldtype'] = 'release'
+
+            a.info['variant'] = '%s-%s-%s' % (a.info.get('plat'),
+                                              a.info.get('arch'),
+                                              a.info.get('bldtype'))
+            if 'toolset' not in a.info:
+                a.info['toolset'] = 'v142'
+
+        # Apply mappings and extract files
+        self.apply_mappings()
+
+        print('Tree extracted to %s' % self.stpath)
+
+        # After creating a bare-bone nupkg layout containing the artifacts
+        # and some spec and props files, call the 'nuget' utility to
+        # make a proper nupkg of it (with all the metadata files).
+        subprocess.check_call("./nuget.sh pack %s -BasePath '%s' -NonInteractive" %  # noqa: E501
+                              (os.path.join(self.stpath,
+                                            'librdkafka.redist.nuspec'),
+                               self.stpath), shell=True)
+
+        return 'librdkafka.redist.%s.nupkg' % vless_version