@netlify/opentelemetry-sdk-setup 1.0.0

package/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2020 Netlify <team@netlify.com>
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,70 @@
+# Opentelemetry SDK Setup
+
+This package extracts the logic necessary to initialise the Opentelemetry JS SDK using our tracing exporter. This not
+only allows us to reuse the initialisation logic across different node process executions but also means **our modules
+don't need to depend on any @opentelemetry module other than the @opentelemetry/api**
+
+## How to use it?
+
+This module is designed to be preloaded via [--import](https://nodejs.org/docs/latest-v18.x/api/cli.html#--importmodule)
+on any node execution. For example:
+
+```
+$> node --import=./lib/bin.js ../build/lib/core/bin.js --debug --tracing.enabled=false --tracing.httpProtocol=https --tracing.host=api.honeycomb.io --tracing.port=443 --tracing.debug=true --tracing.preloadingEnabled=true .
+```
+
+On the script we're instrumenting we can just rely on `@opentelemetry/api` to create spans and interact with the SDK:
+
+```ts
+  import { trace } from '@opentelemetry/api'
+  const tracer = trace.getTracer('secrets-scanning')
+
+  const myInstrumentedFunction = async function() {
+    await tracer.startActiveSpan(
+      'scanning-files',
+      { attributes: { myAttribute: 'foobar' } },
+      async (span) => {
+        doSomeWork()
+        span.end()
+      }
+  }
+
+```
+
+## Sharing and receiving context from outside of the process
+
+Our SDK initialisation is prepared to receive [trace](https://opentelemetry.io/docs/concepts/signals/traces/) and
+[baggage](https://opentelemetry.io/docs/concepts/signals/baggage/) context from outside of the process. This allow us
+to, for example, hook this node process execution to an ongoing trace which is already taking place or share the baggage
+attributes for that execution with the spans created in this process. The list of tracing options show the options
+available to the executable and what they mean.
+
+Unfortunately, to our knowledge, the current `@opentelemetry` setup does not allow us to define an initial global
+context that the root span can inherit from. As a consequence we had to get creative in order to pass the ingested
+attributes to our main script execution, so that the root span can get the newly ingested attributes. We're relying on a
+global property which can be accessed via `@netlify/opentelemetry-utils`. If your process receives any outside
+attributes you can do the following:
+
+```
+$> node --import=./lib/bin.js my-instrumented-script --tracing.httpProtocol=https --tracing.host=api.honeycomb.io --tracing.port=443 --tracing.debug=true --tracing.preloadingEnabled=true --tracing.baggageFilePath='./my-baggage-filepath' --tracing.traceId=<my-trace-id> --tracing.parentSpanId=<the-span-id-of-the-parent>
+```
+
+And on the instrumented script:
+
+```ts
+  import { trace } from '@opentelemetry/api'
+  import { getGlobalContext } from '@netlify/opentelemetry-utils'
+  const tracer = trace.getTracer('secrets-scanning')
+
+  const myInstrumentedFunction = async function() {
+    await tracer.startActiveSpan(
+      'scanning-files',
+      { attributes: { myAttribute: 'foobar' } },
+      getGlobalContext(),
+      async (span) => {
+        doSomeWork()
+        span.end()
+      }
+  }
+
+```

package/bin.js

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+
+// This is a workaround for npm issue: https://github.com/npm/cli/issues/2632
+
+import './lib/bin.js'

package/lib/bin.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/bin.js

@@ -0,0 +1,47 @@
+import process from 'node:process';
+import { diag } from '@opentelemetry/api';
+import argsParser from 'yargs-parser';
+import { startTracing, stopTracing } from './sdk-setup.js';
+import { findExecutablePackageJSON, setGlobalContext } from './util.js';
+const DEFAULT_OTEL_TRACING_PORT = 4317;
+const DEFAULT_OTEL_ENDPOINT_PROTOCOL = 'http';
+const defaultOptions = {
+    preloadingEnabled: false,
+    httpProtocol: DEFAULT_OTEL_ENDPOINT_PROTOCOL,
+    host: 'locahost',
+    port: DEFAULT_OTEL_TRACING_PORT,
+    sampleRate: 1,
+    baggageFilePath: '',
+    apiKey: '-',
+    parentSpanId: '',
+    traceId: '',
+    debug: false,
+};
+const args = argsParser(process.argv);
+// Apply the defaults making sure we're not tripped by falsy values
+const options = Object.entries(defaultOptions)
+    .map(([key, defaultValue]) => {
+    if (args.tracing !== undefined && args.tracing[key] !== undefined) {
+        return { [key]: args.tracing[key] };
+    }
+    return { [key]: defaultValue };
+})
+    .reduce((acc, prop) => ({ ...acc, ...prop }), {});
+const executablePath = args._[1];
+try {
+    const pkg = await findExecutablePackageJSON(executablePath);
+    const rootCtx = await startTracing(options, pkg);
+    if (rootCtx !== undefined) {
+        diag.debug('Setting global root context imported from bagage file');
+        setGlobalContext(rootCtx);
+    }
+    else {
+        diag.debug('Root context undefined, skip setting global root context');
+    }
+}
+catch {
+    // don't blow up the execution in case something fails
+}
+//TODO handle `stopTracing` via `process` event emitter for all the other cases such as
+//SIGINT and SIGTERM signals and potential uncaught exceptions
+process.on('beforeExit', async () => await stopTracing());

package/lib/sdk-setup.d.ts

@@ -0,0 +1,32 @@
+import type { PackageJson } from 'read-pkg-up';
+export type TracingOptions = {
+    /** This is a temporary property to signal preloading is enabled, can be replaced with `enabled` once we retire build's internal sdk setup */
+    preloadingEnabled: boolean;
+    httpProtocol: string;
+    host: string;
+    port: number;
+    /** API Key used for a dedicated trace provider */
+    apiKey: string;
+    /** Sample rate being used for this trace, this allows for consistent probability sampling */
+    sampleRate: number;
+    /** Properties of the root span and trace id used to stitch context */
+    traceId?: string;
+    traceFlags?: number;
+    parentSpanId?: string;
+    baggageFilePath?: string;
+    /** Debug mode enabled - logs to stdout */
+    debug: boolean;
+    /** System log file descriptor */
+    systemLogFile?: number;
+};
+/** Starts the tracing SDK, if there's already a tracing service this will be a no-op */
+export declare const startTracing: (options: TracingOptions, packageJson: PackageJson) => Promise<import("@opentelemetry/api").Context | undefined>;
+/** Stops the tracing service if there's one running. This will flush any ongoing events */
+export declare const stopTracing: () => Promise<void>;
+/** Sets attributes to be propagated across child spans under the current active context
+ * TODO this method will be removed from this package once we move it to a dedicated one to be shared between build,
+ * this setup and any other node module which might use our open telemetry setup
+ * */
+export declare const setMultiSpanAttributes: (attributes: {
+    [key: string]: string;
+}) => import("@opentelemetry/api").Context;

package/lib/sdk-setup.js

@@ -0,0 +1,74 @@
+import { HoneycombSDK } from '@honeycombio/opentelemetry-node';
+import { trace, diag, context, propagation, DiagLogLevel, TraceFlags } from '@opentelemetry/api';
+import { Resource } from '@opentelemetry/resources';
+import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
+import { getDiagLogger, loadBaggageFromFile } from './util.js';
+let sdk;
+/** Starts the tracing SDK, if there's already a tracing service this will be a no-op */
+export const startTracing = async function (options, packageJson) {
+    if (!options.preloadingEnabled)
+        return;
+    if (sdk)
+        return;
+    sdk = new HoneycombSDK({
+        resource: new Resource({
+            [SemanticResourceAttributes.SERVICE_VERSION]: packageJson.version,
+        }),
+        serviceName: packageJson.name,
+        protocol: 'grpc',
+        apiKey: options.apiKey,
+        endpoint: `${options.httpProtocol}://${options.host}:${options.port}`,
+        sampleRate: options.sampleRate,
+        // Turn off auto resource detection so that we fully control the attributes we export
+        autoDetectResources: false,
+    });
+    // Set the diagnostics logger to our system logger. We also need to suppress the override msg
+    // in case there's a default console logger already registered (it would log a msg to it)
+    diag.setLogger(getDiagLogger(options.debug, options.systemLogFile), {
+        logLevel: options.debug ? DiagLogLevel.DEBUG : DiagLogLevel.INFO,
+        suppressOverrideMessage: true,
+    });
+    sdk.start();
+    // Loads the contents of the passed baggageFilePath into the baggage
+    const baggageAttributes = await loadBaggageFromFile(options.baggageFilePath);
+    const baggageCtx = setMultiSpanAttributes(baggageAttributes);
+    const traceFlags = options.traceFlags !== undefined ? options.traceFlags : TraceFlags.NONE;
+    // Sets the current trace ID and span ID based on the options received
+    // this is used as a way to propagate trace context from other processes such as Buildbot
+    if (options.traceId !== undefined && options.parentSpanId !== undefined) {
+        return trace.setSpanContext(baggageCtx, {
+            traceId: options.traceId,
+            spanId: options.parentSpanId,
+            traceFlags: traceFlags,
+            isRemote: true,
+        });
+    }
+    return context.active();
+};
+/** Stops the tracing service if there's one running. This will flush any ongoing events */
+export const stopTracing = async function () {
+    if (!sdk)
+        return;
+    try {
+        // The shutdown method might return an error if we fail to flush the traces
+        // We handle it and use our diagnostics logger
+        await sdk.shutdown();
+        sdk = undefined;
+    }
+    catch (e) {
+        diag.error(e);
+    }
+};
+/** Sets attributes to be propagated across child spans under the current active context
+ * TODO this method will be removed from this package once we move it to a dedicated one to be shared between build,
+ * this setup and any other node module which might use our open telemetry setup
+ * */
+export const setMultiSpanAttributes = function (attributes) {
+    const currentBaggage = propagation.getBaggage(context.active());
+    // Create a baggage if there's none
+    let baggage = currentBaggage === undefined ? propagation.createBaggage() : currentBaggage;
+    Object.entries(attributes).forEach(([key, value]) => {
+        baggage = baggage.setEntry(key, { value });
+    });
+    return propagation.setBaggage(context.active(), baggage);
+};

package/lib/util.d.ts

@@ -0,0 +1,19 @@
+import { DiagLogger, Context } from '@opentelemetry/api';
+import { PackageJson } from 'read-pkg-up';
+/** Given a simple logging function return a `DiagLogger`. Used to setup our system logger as the diag logger.*/
+export declare const getDiagLogger: (debug: boolean, systemLogFile?: number) => DiagLogger;
+export declare const loadBaggageFromFile: (baggageFilePath?: string) => Promise<Record<string, string>>;
+/**
+ * Given a path to a node executable (potentially a symlink) get the module packageJson
+ */
+export declare const findExecutablePackageJSON: (path: string) => Promise<PackageJson>;
+/**
+ * Sets global context to be used when initialising our root span
+ * TODO this will move to a shared package (opentelemetry-utils) to scope the usage of this global property there
+ */
+export declare const setGlobalContext: (ctx: Context) => void;
+/**
+ * Gets the global context to be used when initialising our root span
+ * TODO this will move to a shared package (opentelemetry-utils) to scope the usage of this global property there
+ */
+export declare const getGlobalContext: () => Context;

package/lib/util.js

@@ -0,0 +1,110 @@
+import { createWriteStream } from 'node:fs';
+import { realpath, readFile } from 'node:fs/promises';
+import { diag, context } from '@opentelemetry/api';
+import { parseKeyPairsIntoRecord } from '@opentelemetry/core/build/src/baggage/utils.js';
+import { readPackageUp } from 'read-pkg-up';
+/**
+ * Builds a function for logging data to a provided fileDescriptor (i.e. hidden from
+ * the user-facing build logs)
+ * This has been pulled from @netlify/build as a quick way to hook into the system logger
+ */
+const getSystemLogger = function (debug, 
+/** A system log file descriptor, if non is provided it will be a noop logger */
+systemLogFile) {
+    // If the `debug` flag is used, we return a function that pipes system logs
+    // to the regular logger, as the intention is for them to end up in stdout.
+    // For now we just use plain `console.log`, later on we can revise it
+    if (debug) {
+        return (...args) => console.log(...args);
+    }
+    // If there's not a file descriptor configured for system logs and `debug`
+    // is not set, we return a no-op function that will swallow the errors.
+    if (!systemLogFile) {
+        return () => {
+            // no-op
+        };
+    }
+    // Return a function that writes to the file descriptor configured for system
+    // logs.
+    const fileDescriptor = createWriteStream('', { fd: systemLogFile });
+    fileDescriptor.on('error', () => {
+        console.error('Could not write to system log file');
+    });
+    return (...args) => fileDescriptor.write(`${console.log(...args)}\n`);
+};
+/** Given a simple logging function return a `DiagLogger`. Used to setup our system logger as the diag logger.*/
+export const getDiagLogger = function (debug, 
+/** A system log file descriptor, if non is provided it will be a noop logger */
+systemLogFile) {
+    const logger = getSystemLogger(debug, systemLogFile);
+    const otelLogger = (...args) => {
+        // Debug log msgs can be an array of 1 or 2 elements with the second element being an array fo multiple elements
+        const msgs = args.flat(1);
+        logger('[otel-traces]', ...msgs);
+    };
+    return {
+        debug: otelLogger,
+        info: otelLogger,
+        error: otelLogger,
+        verbose: otelLogger,
+        warn: otelLogger,
+    };
+};
+//** Loads the baggage attributes from a baggage file which follows W3C Baggage specification */
+export const loadBaggageFromFile = async function (baggageFilePath) {
+    if (baggageFilePath === undefined || baggageFilePath.length === 0) {
+        diag.warn('No baggage file path provided, no context loaded');
+        return {};
+    }
+    let baggageString;
+    try {
+        baggageString = await readFile(baggageFilePath, 'utf-8');
+    }
+    catch (error) {
+        diag.error(error);
+        return {};
+    }
+    return parseKeyPairsIntoRecord(baggageString);
+};
+/**
+ * Given a path to a node executable (potentially a symlink) get the module packageJson
+ */
+export const findExecutablePackageJSON = async function (path) {
+    let pathToSearch;
+    try {
+        // resolve symlinks
+        pathToSearch = await realpath(path);
+    }
+    catch {
+        // bail early if we can't resolve the path
+        return {};
+    }
+    try {
+        const result = await readPackageUp({ cwd: pathToSearch, normalize: false });
+        if (result === undefined)
+            return {};
+        const { packageJson } = result;
+        return packageJson;
+    }
+    catch {
+        // packageJson read failed, we ignore the error and return an empty obj
+        return {};
+    }
+};
+/**
+ * Sets global context to be used when initialising our root span
+ * TODO this will move to a shared package (opentelemetry-utils) to scope the usage of this global property there
+ */
+export const setGlobalContext = function (ctx) {
+    global['NETLIFY_GLOBAL_CONTEXT'] = ctx;
+};
+/**
+ * Gets the global context to be used when initialising our root span
+ * TODO this will move to a shared package (opentelemetry-utils) to scope the usage of this global property there
+ */
+export const getGlobalContext = function () {
+    if (global['NETLIFY_GLOBAL_CONTEXT'] === undefined) {
+        return context.active();
+    }
+    return global['NETLIFY_GLOBAL_CONTEXT'];
+};

package/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@netlify/opentelemetry-sdk-setup",
+  "version": "1.0.0",
+  "description": "Opentelemetry SDK setup script",
+  "type": "module",
+  "bin": {
+    "otel-sdk-setup": "./bin.js"
+  },
+  "files": [
+    "bin.js",
+    "lib/**/*"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "build:logos": "vite build",
+    "test": "vitest run",
+    "test:dev": "vitest --ui",
+    "test:ci": "vitest run --reporter=default"
+  },
+  "keywords": [],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/netlify/build.git",
+    "directory": "packages/opentelemetry-sdk-setup"
+  },
+  "bugs": {
+    "url": "https://github.com/netlify/build/issues"
+  },
+  "author": "Netlify Inc.",
+  "dependencies": {
+    "@honeycombio/opentelemetry-node": "^0.6.0",
+    "@opentelemetry/api": "~1.6.0",
+    "@opentelemetry/core": "^1.17.1",
+    "@opentelemetry/resources": "^1.18.1",
+    "@opentelemetry/semantic-conventions": "^1.18.1",
+    "yargs-parser": "^21.1.1"
+  },
+  "devDependencies": {
+    "@types/node": "^14.18.53",
+    "@vitest/coverage-c8": "^0.30.1",
+    "@vitest/ui": "^0.30.1",
+    "typescript": "^5.0.0",
+    "vite": "^4.0.4",
+    "vitest": "^0.30.1"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "gitHead": "9c8c452edb6d062ac7f03eaa64e9a23e0791ad7c"
+}