ddtrace 1.12.1 → 1.18.0

data/ext/ddtrace_profiling_native_extension/http_transport.c

@@ -16,7 +16,6 @@ static ID agent_id; // id of :agent in Ruby
 
 static ID log_failure_to_process_tag_id; // id of :log_failure_to_process_tag in Ruby
 
-static VALUE http_transport_class = Qnil;
 static VALUE library_version_string = Qnil;
 
 struct call_exporter_without_gvl_arguments {
@@ -46,17 +45,18 @@ static VALUE _native_do_export(
   VALUE pprof_data,
   VALUE code_provenance_file_name,
   VALUE code_provenance_data,
-  VALUE tags_as_array
+  VALUE tags_as_array,
+  VALUE internal_metadata_json
 );
 static void *call_exporter_without_gvl(void *call_args);
 static void interrupt_exporter_call(void *cancel_token);
 static VALUE ddtrace_version(void);
 
 void http_transport_init(VALUE profiling_module) {
-  http_transport_class = rb_define_class_under(profiling_module, "HttpTransport", rb_cObject);
+  VALUE http_transport_class = rb_define_class_under(profiling_module, "HttpTransport", rb_cObject);
 
   rb_define_singleton_method(http_transport_class, "_native_validate_exporter",  _native_validate_exporter, 1);
-  rb_define_singleton_method(http_transport_class, "_native_do_export",  _native_do_export, 11);
+  rb_define_singleton_method(http_transport_class, "_native_do_export",  _native_do_export, 12);
 
   ok_symbol = ID2SYM(rb_intern_const("ok"));
   error_symbol = ID2SYM(rb_intern_const("error"));
@@ -179,6 +179,10 @@ static ddog_Vec_Tag convert_tags(VALUE tags_as_array) {
 }
 
 static VALUE log_failure_to_process_tag(VALUE err_details) {
+  VALUE datadog_module = rb_const_get(rb_cObject, rb_intern("Datadog"));
+  VALUE profiling_module = rb_const_get(datadog_module, rb_intern("Profiling"));
+  VALUE http_transport_class = rb_const_get(profiling_module, rb_intern("HttpTransport"));
+
   return rb_funcall(http_transport_class, log_failure_to_process_tag_id, 1, err_details);
 }
 
@@ -200,13 +204,24 @@ static VALUE perform_export(
   ddog_prof_Exporter *exporter,
   ddog_Timespec start,
   ddog_Timespec finish,
-  ddog_prof_Exporter_Slice_File slice_files,
+  ddog_prof_Exporter_Slice_File files_to_compress_and_export,
+  ddog_prof_Exporter_Slice_File files_to_export_unmodified,
   ddog_Vec_Tag *additional_tags,
+  ddog_CharSlice internal_metadata,
   uint64_t timeout_milliseconds
 ) {
   ddog_prof_ProfiledEndpointsStats *endpoints_stats = NULL; // Not in use yet
-  ddog_prof_Exporter_Request_BuildResult build_result =
-    ddog_prof_Exporter_Request_build(exporter, start, finish, slice_files, additional_tags, endpoints_stats, timeout_milliseconds);
+  ddog_prof_Exporter_Request_BuildResult build_result = ddog_prof_Exporter_Request_build(
+    exporter,
+    start,
+    finish,
+    files_to_compress_and_export,
+    files_to_export_unmodified,
+    additional_tags,
+    endpoints_stats,
+    &internal_metadata,
+    timeout_milliseconds
+  );
 
   if (build_result.tag == DDOG_PROF_EXPORTER_REQUEST_BUILD_RESULT_ERR) {
     ddog_prof_Exporter_drop(exporter);
@@ -274,7 +289,8 @@ static VALUE _native_do_export(
   VALUE pprof_data,
   VALUE code_provenance_file_name,
   VALUE code_provenance_data,
-  VALUE tags_as_array
+  VALUE tags_as_array,
+  VALUE internal_metadata_json
 ) {
   ENFORCE_TYPE(upload_timeout_milliseconds, T_FIXNUM);
   ENFORCE_TYPE(start_timespec_seconds, T_FIXNUM);
@@ -284,6 +300,7 @@ static VALUE _native_do_export(
   ENFORCE_TYPE(pprof_file_name, T_STRING);
   ENFORCE_TYPE(pprof_data, T_STRING);
   ENFORCE_TYPE(code_provenance_file_name, T_STRING);
+  ENFORCE_TYPE(internal_metadata_json, T_STRING);
 
   // Code provenance can be disabled and in that case will be set to nil
   bool have_code_provenance = !NIL_P(code_provenance_data);
@@ -296,22 +313,28 @@ static VALUE _native_do_export(
   ddog_Timespec finish =
     {.seconds = NUM2LONG(finish_timespec_seconds), .nanoseconds = NUM2UINT(finish_timespec_nanoseconds)};
 
-  int files_to_report = 1 + (have_code_provenance ? 1 : 0);
-  ddog_prof_Exporter_File files[files_to_report];
-  ddog_prof_Exporter_Slice_File slice_files = {.ptr = files, .len = files_to_report};
+  int to_compress_length = have_code_provenance ? 1 : 0;
+  ddog_prof_Exporter_File to_compress[to_compress_length];
+  int already_compressed_length = 1; // pprof
+  ddog_prof_Exporter_File already_compressed[already_compressed_length];
+
+  ddog_prof_Exporter_Slice_File files_to_compress_and_export = {.ptr = to_compress, .len = to_compress_length};
+  ddog_prof_Exporter_Slice_File files_to_export_unmodified = {.ptr = already_compressed, .len = already_compressed_length};
 
-  files[0] = (ddog_prof_Exporter_File) {
+  already_compressed[0] = (ddog_prof_Exporter_File) {
     .name = char_slice_from_ruby_string(pprof_file_name),
-    .file = byte_slice_from_ruby_string(pprof_data)
+    .file = byte_slice_from_ruby_string(pprof_data),
   };
+
   if (have_code_provenance) {
-    files[1] = (ddog_prof_Exporter_File) {
+    to_compress[0] = (ddog_prof_Exporter_File) {
       .name = char_slice_from_ruby_string(code_provenance_file_name),
-      .file = byte_slice_from_ruby_string(code_provenance_data)
+      .file = byte_slice_from_ruby_string(code_provenance_data),
     };
   }
 
   ddog_Vec_Tag *null_additional_tags = NULL;
+  ddog_CharSlice internal_metadata = char_slice_from_ruby_string(internal_metadata_json);
 
   ddog_prof_Exporter_NewResult exporter_result = create_exporter(exporter_configuration, tags_as_array);
   // Note: Do not add anything that can raise exceptions after this line, as otherwise the exporter memory will leak
@@ -319,7 +342,16 @@ static VALUE _native_do_export(
   VALUE failure_tuple = handle_exporter_failure(exporter_result);
   if (!NIL_P(failure_tuple)) return failure_tuple;
 
-  return perform_export(exporter_result.ok, start, finish, slice_files, null_additional_tags, timeout_milliseconds);
+  return perform_export(
+    exporter_result.ok,
+    start,
+    finish,
+    files_to_compress_and_export,
+    files_to_export_unmodified,
+    null_additional_tags,
+    internal_metadata,
+    timeout_milliseconds
+  );
 }
 
 static void *call_exporter_without_gvl(void *call_args) {

data/ext/ddtrace_profiling_native_extension/libdatadog_helpers.c

@@ -0,0 +1,42 @@
+#include "libdatadog_helpers.h"
+
+#include <ruby.h>
+
+const char *ruby_value_type_to_string(enum ruby_value_type type) {
+  return ruby_value_type_to_char_slice(type).ptr;
+}
+
+ddog_CharSlice ruby_value_type_to_char_slice(enum ruby_value_type type) {
+  switch (type) {
+    case(RUBY_T_NONE    ): return DDOG_CHARSLICE_C("T_NONE");
+    case(RUBY_T_OBJECT  ): return DDOG_CHARSLICE_C("T_OBJECT");
+    case(RUBY_T_CLASS   ): return DDOG_CHARSLICE_C("T_CLASS");
+    case(RUBY_T_MODULE  ): return DDOG_CHARSLICE_C("T_MODULE");
+    case(RUBY_T_FLOAT   ): return DDOG_CHARSLICE_C("T_FLOAT");
+    case(RUBY_T_STRING  ): return DDOG_CHARSLICE_C("T_STRING");
+    case(RUBY_T_REGEXP  ): return DDOG_CHARSLICE_C("T_REGEXP");
+    case(RUBY_T_ARRAY   ): return DDOG_CHARSLICE_C("T_ARRAY");
+    case(RUBY_T_HASH    ): return DDOG_CHARSLICE_C("T_HASH");
+    case(RUBY_T_STRUCT  ): return DDOG_CHARSLICE_C("T_STRUCT");
+    case(RUBY_T_BIGNUM  ): return DDOG_CHARSLICE_C("T_BIGNUM");
+    case(RUBY_T_FILE    ): return DDOG_CHARSLICE_C("T_FILE");
+    case(RUBY_T_DATA    ): return DDOG_CHARSLICE_C("T_DATA");
+    case(RUBY_T_MATCH   ): return DDOG_CHARSLICE_C("T_MATCH");
+    case(RUBY_T_COMPLEX ): return DDOG_CHARSLICE_C("T_COMPLEX");
+    case(RUBY_T_RATIONAL): return DDOG_CHARSLICE_C("T_RATIONAL");
+    case(RUBY_T_NIL     ): return DDOG_CHARSLICE_C("T_NIL");
+    case(RUBY_T_TRUE    ): return DDOG_CHARSLICE_C("T_TRUE");
+    case(RUBY_T_FALSE   ): return DDOG_CHARSLICE_C("T_FALSE");
+    case(RUBY_T_SYMBOL  ): return DDOG_CHARSLICE_C("T_SYMBOL");
+    case(RUBY_T_FIXNUM  ): return DDOG_CHARSLICE_C("T_FIXNUM");
+    case(RUBY_T_UNDEF   ): return DDOG_CHARSLICE_C("T_UNDEF");
+    case(RUBY_T_IMEMO   ): return DDOG_CHARSLICE_C("T_IMEMO");
+    case(RUBY_T_NODE    ): return DDOG_CHARSLICE_C("T_NODE");
+    case(RUBY_T_ICLASS  ): return DDOG_CHARSLICE_C("T_ICLASS");
+    case(RUBY_T_ZOMBIE  ): return DDOG_CHARSLICE_C("T_ZOMBIE");
+    #ifndef NO_T_MOVED
+    case(RUBY_T_MOVED   ): return DDOG_CHARSLICE_C("T_MOVED");
+    #endif
+                  default: return DDOG_CHARSLICE_C("BUG: Unknown value for ruby_value_type");
+  }
+}

data/ext/ddtrace_profiling_native_extension/libdatadog_helpers.h

@@ -23,3 +23,9 @@ inline static VALUE get_error_details_and_drop(ddog_Error *error) {
   ddog_Error_drop(error);
   return result;
 }
+
+// Used for pretty printing this Ruby enum. Returns "T_UNKNOWN_OR_MISSING_RUBY_VALUE_TYPE_ENTRY" for unknown elements.
+// In practice, there's a few types that the profiler will probably never encounter, but I've added all entries of
+// ruby_value_type that Ruby uses so that we can also use this for debugging.
+const char *ruby_value_type_to_string(enum ruby_value_type type);
+ddog_CharSlice ruby_value_type_to_char_slice(enum ruby_value_type type);

data/ext/ddtrace_profiling_native_extension/native_extension_helpers.rb

@@ -15,7 +15,7 @@ module Datadog
       # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on debase-ruby_core_source
       CAN_USE_MJIT_HEADER = RUBY_VERSION.start_with?('2.6', '2.7', '3.0.', '3.1.', '3.2.')
 
-      LIBDATADOG_VERSION = '~> 2.0.0.1.0'
+      LIBDATADOG_VERSION = '~> 5.0.0.1.0'
 
       def self.fail_install_if_missing_extension?
         ENV[ENV_FAIL_INSTALL_IF_MISSING_EXTENSION].to_s.strip.downcase == 'true'

data/ext/ddtrace_profiling_native_extension/pid_controller.c

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2023 Datadog, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Originally imported from https://github.com/DataDog/java-profiler/blob/11fe6206c31a14c6e5134e8401eaec8b22c618d7/ddprof-lib/src/main/cpp/pidController.cpp
+
+#include "pid_controller.h"
+
+#include <math.h>
+
+inline static double computeAlpha(float cutoff) {
+    if (cutoff <= 0)
+        return 1;
+    // α(fₙ) = cos(2πfₙ) - 1 + √( cos(2πfₙ)² - 4 cos(2πfₙ) + 3 )
+    const double c = cos(2 * ((double) M_PI) * cutoff);
+    return c - 1 + sqrt(c * c - 4 * c + 3);
+}
+
+void pid_controller_init(pid_controller *controller, u64 target_per_second, double proportional_gain, double integral_gain, double derivative_gain, int sampling_window, double cutoff_secs) {
+    controller->_target = target_per_second * sampling_window;
+    controller->_proportional_gain = proportional_gain;
+    controller->_integral_gain = integral_gain * sampling_window;
+    controller->_derivative_gain = derivative_gain / sampling_window;
+    controller->_alpha = computeAlpha(sampling_window / cutoff_secs);
+    controller->_avg_error= 0;
+    controller->_integral_value = 0;
+}
+
+double pid_controller_compute(pid_controller *controller, u64 input, double time_delta_coefficient) {
+    // time_delta_coefficient allows variable sampling window
+    // the values are linearly scaled using that coefficient to reinterpret the given value within the expected sampling window
+    double absolute_error = (((double) controller->_target) - ((double) input)) * time_delta_coefficient;
+
+    double avg_error = (controller->_alpha * absolute_error) + ((1 - controller->_alpha) * controller->_avg_error);
+    double derivative = avg_error - controller->_avg_error;
+
+    // PID formula:
+    // u[k] = Kp e[k] + Ki e_i[k] + Kd e_d[k], control signal
+    double signal = controller->_proportional_gain * absolute_error + controller->_integral_gain * controller->_integral_value + controller->_derivative_gain * derivative;
+
+    controller->_integral_value += absolute_error;
+    controller->_avg_error = avg_error;
+
+    return signal;
+}

data/ext/ddtrace_profiling_native_extension/pid_controller.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2023 Datadog, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Originally imported from https://github.com/DataDog/java-profiler/blob/11fe6206c31a14c6e5134e8401eaec8b22c618d7/ddprof-lib/src/main/cpp/pidController.h
+
+#ifndef _PIDCONTROLLER_H
+#define _PIDCONTROLLER_H
+
+// From arch.h in java-profiler
+typedef unsigned long long u64;
+
+/*
+ * A simple implementation of a PID controller.
+ * Heavily influenced by https://tttapa.github.io/Pages/Arduino/Control-Theory/Motor-Fader/PID-Cpp-Implementation.html 
+ */
+
+typedef struct {
+        u64 _target;
+        double _proportional_gain;
+        double _derivative_gain;
+        double _integral_gain;
+        double _alpha;
+
+        double _avg_error;
+        long long _integral_value;
+} pid_controller;
+
+void pid_controller_init(pid_controller *controller, u64 target_per_second, double proportional_gain, double integral_gain, double derivative_gain, int sampling_window, double cutoff_secs);
+        
+double pid_controller_compute(pid_controller *controller, u64 input, double time_delta_seconds);
+
+#endif

data/ext/ddtrace_profiling_native_extension/private_vm_api_access.c

@@ -58,9 +58,12 @@ static inline rb_thread_t *thread_struct_from_object(VALUE thread) {
 }
 
 rb_nativethread_id_t pthread_id_for(VALUE thread) {
-  // struct rb_native_thread was introduced in Ruby 3.2 (preview2): https://github.com/ruby/ruby/pull/5836
+  // struct rb_native_thread was introduced in Ruby 3.2: https://github.com/ruby/ruby/pull/5836
   #ifndef NO_RB_NATIVE_THREAD
-    return thread_struct_from_object(thread)->nt->thread_id;
+    struct rb_native_thread* native_thread = thread_struct_from_object(thread)->nt;
+    // This can be NULL on Ruby 3.3 with MN threads (RUBY_MN_THREADS=1)
+    if (native_thread == NULL) return 0;
+    return native_thread->thread_id;
   #else
     return thread_struct_from_object(thread)->thread_id;
   #endif
@@ -82,6 +85,16 @@ bool is_current_thread_holding_the_gvl(void) {
   return owner.valid && pthread_equal(pthread_self(), owner.owner);
 }
 
+#ifdef HAVE_RUBY_RACTOR_H
+  static inline rb_ractor_t *ddtrace_get_ractor(void) {
+    #ifndef USE_RACTOR_INTERNAL_APIS_DIRECTLY // Ruby >= 3.3
+      return thread_struct_from_object(rb_thread_current())->ractor;
+    #else
+      return GET_RACTOR();
+    #endif
+  }
+#endif
+
 #ifndef NO_GVL_OWNER // Ruby < 2.6 doesn't have the owner/running field
   // NOTE: Reading the owner in this is a racy read, because we're not grabbing the lock that Ruby uses to protect it.
   //
@@ -94,24 +107,25 @@ bool is_current_thread_holding_the_gvl(void) {
   current_gvl_owner gvl_owner(void) {
     const rb_thread_t *current_owner =
       #ifndef NO_RB_THREAD_SCHED // Introduced in Ruby 3.2 as a replacement for struct rb_global_vm_lock_struct
-        GET_RACTOR()->threads.sched.running;
+        ddtrace_get_ractor()->threads.sched.running;
       #elif HAVE_RUBY_RACTOR_H
-        GET_RACTOR()->threads.gvl.owner;
+        ddtrace_get_ractor()->threads.gvl.owner;
       #else
         GET_VM()->gvl.owner;
       #endif
 
     if (current_owner == NULL) return (current_gvl_owner) {.valid = false};
 
-    return (current_gvl_owner) {
-      .valid = true,
-      .owner =
-        #ifndef NO_RB_NATIVE_THREAD
-          current_owner->nt->thread_id
-        #else
-          current_owner->thread_id
-        #endif
-    };
+    #ifndef NO_RB_NATIVE_THREAD
+      struct rb_native_thread* current_owner_native_thread = current_owner->nt;
+
+      // This can be NULL on Ruby 3.3 with MN threads (RUBY_MN_THREADS=1)
+      if (current_owner_native_thread == NULL) return (current_gvl_owner) {.valid = false};
+
+      return (current_gvl_owner) {.valid = true, .owner = current_owner_native_thread->thread_id};
+    #else
+      return (current_gvl_owner) {.valid = true, .owner = current_owner->thread_id};
+    #endif
   }
 #else
   current_gvl_owner gvl_owner(void) {
@@ -172,7 +186,9 @@ uint64_t native_thread_id_for(VALUE thread) {
   // The tid is only available on Ruby >= 3.1 + Linux (and FreeBSD). It's the same as `gettid()` aka the task id as seen in /proc
   #if !defined(NO_THREAD_TID) && defined(RB_THREAD_T_HAS_NATIVE_ID)
     #ifndef NO_RB_NATIVE_THREAD
-      return thread_struct_from_object(thread)->nt->tid;
+      struct rb_native_thread* native_thread = thread_struct_from_object(thread)->nt;
+      if (native_thread == NULL) rb_raise(rb_eRuntimeError, "BUG: rb_native_thread* is null. Is this Ruby running with RUBY_MN_THREADS=1?");
+      return native_thread->tid;
     #else
       return thread_struct_from_object(thread)->tid;
     #endif
@@ -234,7 +250,7 @@ void ddtrace_thread_list(VALUE result_array) {
   // I suspect the design in `rb_ractor_thread_list` may be done that way to perhaps in the future expose it to be
   // called from a different Ractor, but I'm not sure...
   #ifdef HAVE_RUBY_RACTOR_H
-    rb_ractor_t *current_ractor = GET_RACTOR();
+    rb_ractor_t *current_ractor = ddtrace_get_ractor();
     ccan_list_for_each(&current_ractor->threads.set, thread, lt_node) {
   #else
     rb_vm_t *vm =
@@ -302,7 +318,8 @@ VALUE thread_name_for(VALUE thread) {
 // Taken from upstream vm_backtrace.c at commit 5f10bd634fb6ae8f74a4ea730176233b0ca96954 (March 2022, Ruby 3.2 trunk)
 // Copyright (C) 1993-2012 Yukihiro Matsumoto
 // to support our custom rb_profile_frames (see below)
-// Modifications: None
+// Modifications:
+// * Support int first_lineno for Ruby 3.2.0+ (https://github.com/ruby/ruby/pull/6430)
 //
 // `node_id` gets used depending on Ruby VM compilation settings (USE_ISEQ_NODE_ID being defined).
 // To avoid getting false "unused argument" warnings in setups where it's not used, we need to do this weird dance
@@ -322,7 +339,11 @@ calc_pos(const rb_iseq_t *iseq, const VALUE *pc, int *lineno, int *node_id)
             VM_ASSERT(! ISEQ_BODY(iseq)->local_table_size);
             return 0;
         }
-        if (lineno) *lineno = FIX2INT(ISEQ_BODY(iseq)->location.first_lineno);
+        # ifndef NO_INT_FIRST_LINENO // Ruby 3.2+
+          if (lineno) *lineno = ISEQ_BODY(iseq)->location.first_lineno;
+        # else
+          if (lineno) *lineno = FIX2INT(ISEQ_BODY(iseq)->location.first_lineno);
+        #endif
 #ifdef USE_ISEQ_NODE_ID
         if (node_id) *node_id = -1;
 #endif
@@ -392,6 +413,7 @@ calc_lineno(const rb_iseq_t *iseq, const VALUE *pc)
 //   the `VALUE` returned by rb_profile_frames returns `(eval)` instead of the path of the file where the `eval`
 //   was called from.
 // * Imported fix from https://github.com/ruby/ruby/pull/7116 to avoid sampling threads that are still being created
+// * Imported fix from https://github.com/ruby/ruby/pull/8415 to avoid potential crash when using YJIT.
 //
 // What is rb_profile_frames?
 // `rb_profile_frames` is a Ruby VM debug API added for use by profilers for sampling the stack trace of a Ruby thread.
@@ -427,12 +449,15 @@ int ddtrace_rb_profile_frames(VALUE thread, int start, int limit, VALUE *buff, i
     // Modified from upstream: Instead of using `GET_EC` to collect info from the current thread,
     // support sampling any thread (including the current) passed as an argument
     rb_thread_t *th = thread_struct_from_object(thread);
-#ifndef USE_THREAD_INSTEAD_OF_EXECUTION_CONTEXT // Modern Rubies
-    const rb_execution_context_t *ec = th->ec;
-#else // Ruby < 2.5
-    const rb_thread_t *ec = th;
-#endif
+    #ifndef USE_THREAD_INSTEAD_OF_EXECUTION_CONTEXT // Modern Rubies
+      const rb_execution_context_t *ec = th->ec;
+    #else // Ruby < 2.5
+      const rb_thread_t *ec = th;
+    #endif
     const rb_control_frame_t *cfp = ec->cfp, *end_cfp = RUBY_VM_END_CONTROL_FRAME(ec);
+    #ifndef NO_JIT_RETURN
+      const rb_control_frame_t *top = cfp;
+    #endif
     const rb_callable_method_entry_t *cme;
 
     // Avoid sampling dead threads
@@ -507,7 +532,20 @@ int ddtrace_rb_profile_frames(VALUE thread, int start, int limit, VALUE *buff, i
                 buff[i] = (VALUE)cfp->iseq;
             }
 
-            lines[i] = calc_lineno(cfp->iseq, cfp->pc);
+            // The topmost frame may not have an updated PC because the JIT
+            // may not have set one.  The JIT compiler will update the PC
+            // before entering a new function (so that `caller` will work),
+            // so only the topmost frame could possibly have an out of date PC
+            #ifndef NO_JIT_RETURN
+              if (cfp == top && cfp->jit_return) {
+                lines[i] = 0;
+              } else {
+                lines[i] = calc_lineno(cfp->iseq, cfp->pc);
+              }
+            #else // Ruby < 3.1
+              lines[i] = calc_lineno(cfp->iseq, cfp->pc);
+            #endif
+
             is_ruby_frame[i] = true;
             i++;
         }
@@ -731,15 +769,10 @@ check_method_entry(VALUE obj, int can_be_svar)
     // versions, so we need to do a bit more work.
     struct rb_ractor_struct *ruby_single_main_ractor = NULL;
 
-    // Taken from upstream ractor.c at commit a1b01e7701f9fc370f8dff777aad6d39a2c5a3e3 (May 2023, Ruby 3.3.0-preview1)
-    // to allow us to ensure that we're always operating on the main ractor (if Ruby has ractors)
-    // Modifications:
-    // * None
-    bool rb_ractor_main_p_(void)
-    {
-        VM_ASSERT(rb_multi_ractor_p());
-        rb_execution_context_t *ec = GET_EC();
-        return rb_ec_ractor_ptr(ec) == rb_ec_vm_ptr(ec)->ractor.main_ractor;
+    // Alternative implementation of rb_ractor_main_p_ that avoids relying on non-public symbols
+    bool rb_ractor_main_p_(void) {
+      // We need to get the main ractor in a bit of a roundabout way, since Ruby >= 3.3 hid `GET_VM()`
+      return ddtrace_get_ractor() == thread_struct_from_object(rb_thread_current())->vm->ractor.main_ractor;
     }
   #else
     // Directly access Ruby internal fast path for detecting multiple Ractors.
@@ -767,3 +800,74 @@ check_method_entry(VALUE obj, int can_be_svar)
   // they're always on the main Ractor
   bool ddtrace_rb_ractor_main_p(void) { return true; }
 #endif // NO_RACTORS
+
+// This is a tweaked and inlined version of
+// threadptr_invoke_proc_location + rb_proc_location + iseq_location .
+//
+// It's useful to have here because not all of the methods above are accessible to extensions + to avoid the
+// array allocation that iseq_location did to contain its return value.
+static const rb_iseq_t *maybe_thread_invoke_proc_iseq(VALUE thread_value) {
+  rb_thread_t *thread = thread_struct_from_object(thread_value);
+
+  #ifndef NO_THREAD_INVOKE_ARG // Ruby 2.6+
+    if (thread->invoke_type != thread_invoke_type_proc) return NULL;
+
+    VALUE proc = thread->invoke_arg.proc.proc;
+  #else
+    if (thread->first_func || !thread->first_proc) return NULL;
+
+    VALUE proc = thread->first_proc;
+  #endif
+
+  const rb_iseq_t *iseq = rb_proc_get_iseq(proc, 0);
+  if (iseq == NULL) return NULL;
+
+  rb_iseq_check(iseq);
+  return iseq;
+}
+
+VALUE invoke_location_for(VALUE thread, int *line_location) {
+  const rb_iseq_t *iseq = maybe_thread_invoke_proc_iseq(thread);
+
+  if (iseq == NULL) return Qnil;
+
+  *line_location = NUM2INT(rb_iseq_first_lineno(iseq));
+  return rb_iseq_path(iseq);
+}
+
+void self_test_mn_enabled(void) {
+  #ifdef NO_MN_THREADS_AVAILABLE
+    return;
+  #else
+    if (ddtrace_get_ractor()->threads.sched.enable_mn_threads == true) {
+      rb_raise(rb_eRuntimeError, "Ruby VM is running with RUBY_MN_THREADS=1. This is not yet supported");
+    }
+  #endif
+}
+
+// Taken from upstream imemo.h at commit 6ebcf25de2859b5b6402b7e8b181066c32d0e0bf (November 2023, master branch)
+// (See the Ruby project copyright and license above)
+// to enable calling rb_imemo_name
+//
+// Modifications:
+// * Added IMEMO_MASK define
+// * Changed return type to int to avoid having to define `enum imemo_type`
+static inline int ddtrace_imemo_type(VALUE imemo) {
+  // This mask is the same between Ruby 2.5 and 3.3-preview3. Furthermore, the intention of this method is to be used
+  // to call `rb_imemo_name` which correctly handles invalid numbers so even if the mask changes in the future, at most
+  // we'll get incorrect results (and never a VM crash)
+  #define IMEMO_MASK   0x0f
+  return (RBASIC(imemo)->flags >> FL_USHIFT) & IMEMO_MASK;
+}
+
+// Safety: This function assumes the object passed in is of the imemo type. But in the worst case, you'll just get
+// a string that doesn't make any sense.
+#ifndef NO_IMEMO_NAME
+  const char *imemo_kind(VALUE imemo) {
+    return rb_imemo_name(ddtrace_imemo_type(imemo));
+  }
+#else
+  const char *imemo_kind(__attribute__((unused)) VALUE imemo) {
+    return NULL;
+  }
+#endif

data/ext/ddtrace_profiling_native_extension/private_vm_api_access.h

@@ -43,3 +43,15 @@ bool ddtrace_rb_ractor_main_p(void);
 
 // See comment on `record_placeholder_stack_in_native_code` for a full explanation of what this means (and why we don't just return 0)
 #define PLACEHOLDER_STACK_IN_NATIVE_CODE -1
+
+// This method provides the file and line of the "invoke location" of a thread (first file:line of the block used to
+// start the thread), if any.
+// This is what Ruby shows in `Thread#to_s`.
+// The file is returned directly, and the line is recorded onto *line_location.
+VALUE invoke_location_for(VALUE thread, int *line_location);
+
+// Check if RUBY_MN_THREADS is enabled (aka main Ractor is not doing 1:1 threads)
+void self_test_mn_enabled(void);
+
+// Provides more specific information on what kind an imemo is
+const char *imemo_kind(VALUE imemo);

data/ext/ddtrace_profiling_native_extension/profiling.c

@@ -41,8 +41,6 @@ void DDTRACE_EXPORT Init_ddtrace_profiling_native_extension(void) {
   rb_define_singleton_method(native_extension_module, "native_working?", native_working_p, 0);
   rb_funcall(native_extension_module, rb_intern("private_class_method"), 1, ID2SYM(rb_intern("native_working?")));
 
-  rb_define_singleton_method(native_extension_module, "clock_id_for", clock_id_for, 1); // from clock_id.h
-
   collectors_cpu_and_wall_time_worker_init(profiling_module);
   collectors_dynamic_sampling_rate_init(profiling_module);
   collectors_idle_sampling_helper_init(profiling_module);
@@ -70,6 +68,7 @@ void DDTRACE_EXPORT Init_ddtrace_profiling_native_extension(void) {
 
 static VALUE native_working_p(DDTRACE_UNUSED VALUE _self) {
   self_test_clock_id();
+  self_test_mn_enabled();
 
   return Qtrue;
 }