debase-ruby_core_source 4.0.1 → 4.1.0.preview1

data/lib/debase/ruby_core_source/ruby-4.1.0-preview1/vm_callinfo.h

@@ -0,0 +1,688 @@
+#ifndef RUBY_VM_CALLINFO_H                               /*-*-C-*-vi:se ft=c:*/
+#define RUBY_VM_CALLINFO_H
+/**
+ * @author     Ruby developers <ruby-core@ruby-lang.org>
+ * @copyright  This  file  is   a  part  of  the   programming  language  Ruby.
+ *             Permission  is hereby  granted,  to  either redistribute  and/or
+ *             modify this file, provided that  the conditions mentioned in the
+ *             file COPYING are met.  Consult the file for details.
+ */
+
+#include "debug_counter.h"
+#include "internal/class.h"
+#include "shape.h"
+
+enum vm_call_flag_bits {
+    VM_CALL_ARGS_SPLAT_bit,     // m(*args)
+    VM_CALL_ARGS_BLOCKARG_bit,  // m(&block)
+    VM_CALL_FCALL_bit,          // m(args)   # receiver is self
+    VM_CALL_VCALL_bit,          // m         # method call that looks like a local variable
+    VM_CALL_ARGS_SIMPLE_bit,    // !(ci->flag & (SPLAT|BLOCKARG|KWARG|KW_SPLAT|FORWARDING)) && !has_block_iseq
+    VM_CALL_KWARG_bit,          // has kwarg
+    VM_CALL_KW_SPLAT_bit,       // m(**opts)
+    VM_CALL_TAILCALL_bit,       // located at tail position
+    VM_CALL_SUPER_bit,          // super
+    VM_CALL_ZSUPER_bit,         // zsuper
+    VM_CALL_OPT_SEND_bit,       // internal flag
+    VM_CALL_KW_SPLAT_MUT_bit,   // kw splat hash can be modified (to avoid allocating a new one)
+    VM_CALL_ARGS_SPLAT_MUT_bit, // args splat can be modified (to avoid allocating a new one)
+    VM_CALL_FORWARDING_bit,     // m(...)
+    VM_CALL__END
+};
+
+#define VM_CALL_ARGS_SPLAT      (0x01 << VM_CALL_ARGS_SPLAT_bit)
+#define VM_CALL_ARGS_BLOCKARG   (0x01 << VM_CALL_ARGS_BLOCKARG_bit)
+#define VM_CALL_FCALL           (0x01 << VM_CALL_FCALL_bit)
+#define VM_CALL_VCALL           (0x01 << VM_CALL_VCALL_bit)
+#define VM_CALL_ARGS_SIMPLE     (0x01 << VM_CALL_ARGS_SIMPLE_bit)
+#define VM_CALL_KWARG           (0x01 << VM_CALL_KWARG_bit)
+#define VM_CALL_KW_SPLAT        (0x01 << VM_CALL_KW_SPLAT_bit)
+#define VM_CALL_TAILCALL        (0x01 << VM_CALL_TAILCALL_bit)
+#define VM_CALL_SUPER           (0x01 << VM_CALL_SUPER_bit)
+#define VM_CALL_ZSUPER          (0x01 << VM_CALL_ZSUPER_bit)
+#define VM_CALL_OPT_SEND        (0x01 << VM_CALL_OPT_SEND_bit)
+#define VM_CALL_KW_SPLAT_MUT    (0x01 << VM_CALL_KW_SPLAT_MUT_bit)
+#define VM_CALL_ARGS_SPLAT_MUT  (0x01 << VM_CALL_ARGS_SPLAT_MUT_bit)
+#define VM_CALL_FORWARDING      (0x01 << VM_CALL_FORWARDING_bit)
+
+struct rb_callinfo_kwarg {
+    int keyword_len;
+    rb_atomic_t references;
+    VALUE keywords[];
+};
+
+static inline size_t
+rb_callinfo_kwarg_bytes(int keyword_len)
+{
+    return rb_size_mul_add_or_raise(
+        keyword_len,
+        sizeof(VALUE),
+        sizeof(struct rb_callinfo_kwarg),
+        rb_eRuntimeError);
+}
+
+static inline void
+rb_callinfo_kwarg_retain(struct rb_callinfo_kwarg *kwarg)
+{
+    if (kwarg) RUBY_ATOMIC_INC(kwarg->references);
+}
+
+static inline void
+rb_callinfo_kwarg_release(struct rb_callinfo_kwarg *kwarg)
+{
+    if (kwarg && RUBY_ATOMIC_FETCH_SUB(kwarg->references, 1) == 1) {
+        ruby_xfree_sized(kwarg, rb_callinfo_kwarg_bytes(kwarg->keyword_len));
+    }
+}
+
+// imemo_callinfo
+struct rb_callinfo {
+    VALUE flags;
+    const struct rb_callinfo_kwarg *kwarg;
+    VALUE mid;
+    unsigned int flag;
+    unsigned int argc;
+};
+
+#if !defined(USE_EMBED_CI) || (USE_EMBED_CI+0)
+#undef USE_EMBED_CI
+#define USE_EMBED_CI 1
+#else
+#undef USE_EMBED_CI
+#define USE_EMBED_CI 0
+#endif
+
+#if SIZEOF_VALUE == 8
+#define CI_EMBED_TAG_bits   1
+#define CI_EMBED_ARGC_bits 15
+#define CI_EMBED_FLAG_bits 16
+#define CI_EMBED_ID_bits   32
+#elif SIZEOF_VALUE == 4
+#define CI_EMBED_TAG_bits   1
+#define CI_EMBED_ARGC_bits  3
+#define CI_EMBED_FLAG_bits 13
+#define CI_EMBED_ID_bits   15
+#endif
+
+#if (CI_EMBED_TAG_bits + CI_EMBED_ARGC_bits + CI_EMBED_FLAG_bits + CI_EMBED_ID_bits) != (SIZEOF_VALUE * 8)
+#error
+#endif
+
+#define CI_EMBED_FLAG 0x01
+#define CI_EMBED_ARGC_SHFT (CI_EMBED_TAG_bits)
+#define CI_EMBED_ARGC_MASK ((((VALUE)1)<<CI_EMBED_ARGC_bits) - 1)
+#define CI_EMBED_FLAG_SHFT (CI_EMBED_TAG_bits + CI_EMBED_ARGC_bits)
+#define CI_EMBED_FLAG_MASK ((((VALUE)1)<<CI_EMBED_FLAG_bits) - 1)
+#define CI_EMBED_ID_SHFT   (CI_EMBED_TAG_bits + CI_EMBED_ARGC_bits + CI_EMBED_FLAG_bits)
+#define CI_EMBED_ID_MASK   ((((VALUE)1)<<CI_EMBED_ID_bits) - 1)
+
+static inline bool
+vm_ci_packed_p(const struct rb_callinfo *ci)
+{
+    if (!USE_EMBED_CI) {
+        return 0;
+    }
+    if (LIKELY(((VALUE)ci) & 0x01)) {
+        return 1;
+    }
+    else {
+        VM_ASSERT(IMEMO_TYPE_P(ci, imemo_callinfo));
+        return 0;
+    }
+}
+
+static inline bool
+vm_ci_p(const struct rb_callinfo *ci)
+{
+    if (vm_ci_packed_p(ci) || IMEMO_TYPE_P(ci, imemo_callinfo)) {
+        return 1;
+    }
+    else {
+        return 0;
+    }
+}
+
+static inline ID
+vm_ci_mid(const struct rb_callinfo *ci)
+{
+    if (vm_ci_packed_p(ci)) {
+        return (((VALUE)ci) >> CI_EMBED_ID_SHFT) & CI_EMBED_ID_MASK;
+    }
+    else {
+        return (ID)ci->mid;
+    }
+}
+
+static inline unsigned int
+vm_ci_flag(const struct rb_callinfo *ci)
+{
+    if (vm_ci_packed_p(ci)) {
+        return (unsigned int)((((VALUE)ci) >> CI_EMBED_FLAG_SHFT) & CI_EMBED_FLAG_MASK);
+    }
+    else {
+        return ci->flag;
+    }
+}
+
+static inline unsigned int
+vm_ci_argc(const struct rb_callinfo *ci)
+{
+    if (vm_ci_packed_p(ci)) {
+        return (unsigned int)((((VALUE)ci) >> CI_EMBED_ARGC_SHFT) & CI_EMBED_ARGC_MASK);
+    }
+    else {
+        return ci->argc;
+    }
+}
+
+static inline const struct rb_callinfo_kwarg *
+vm_ci_kwarg(const struct rb_callinfo *ci)
+{
+    if (vm_ci_packed_p(ci)) {
+        return NULL;
+    }
+    else {
+        return ci->kwarg;
+    }
+}
+
+static inline void
+vm_ci_dump(const struct rb_callinfo *ci)
+{
+    if (vm_ci_packed_p(ci)) {
+        ruby_debug_printf("packed_ci ID:%s flag:%x argc:%u\n",
+                          rb_id2name(vm_ci_mid(ci)), vm_ci_flag(ci), vm_ci_argc(ci));
+    }
+    else {
+        rp(ci);
+    }
+}
+
+#define vm_ci_new(mid, flag, argc, kwarg) vm_ci_new_(mid, flag, argc, kwarg, __FILE__, __LINE__)
+#define vm_ci_new_runtime(mid, flag, argc, kwarg) vm_ci_new_runtime_(mid, flag, argc, kwarg, __FILE__, __LINE__)
+
+/* This is passed to STATIC_ASSERT.  Cannot be an inline function. */
+#define VM_CI_EMBEDDABLE_P(mid, flag, argc, kwarg) \
+    (((mid ) & ~CI_EMBED_ID_MASK)   ? false :      \
+     ((flag) & ~CI_EMBED_FLAG_MASK) ? false :      \
+     ((argc) & ~CI_EMBED_ARGC_MASK) ? false :      \
+      (kwarg)                       ? false : true)
+
+#define vm_ci_new_id(mid, flag, argc, must_zero) \
+    ((const struct rb_callinfo *)                \
+     ((((VALUE)(mid )) << CI_EMBED_ID_SHFT)   |  \
+      (((VALUE)(flag)) << CI_EMBED_FLAG_SHFT) |  \
+      (((VALUE)(argc)) << CI_EMBED_ARGC_SHFT) |  \
+      RUBY_FIXNUM_FLAG))
+
+// vm_method.c
+const struct rb_callinfo *rb_vm_ci_lookup(ID mid, unsigned int flag, unsigned int argc, const struct rb_callinfo_kwarg *kwarg);
+void rb_vm_ci_free(const struct rb_callinfo *);
+
+static inline const struct rb_callinfo *
+vm_ci_new_(ID mid, unsigned int flag, unsigned int argc, const struct rb_callinfo_kwarg *kwarg, const char *file, int line)
+{
+    if (USE_EMBED_CI && VM_CI_EMBEDDABLE_P(mid, flag, argc, kwarg)) {
+        RB_DEBUG_COUNTER_INC(ci_packed);
+        return vm_ci_new_id(mid, flag, argc, kwarg);
+    }
+
+    const bool debug = 0;
+    if (debug) ruby_debug_printf("%s:%d ", file, line);
+
+    const struct rb_callinfo *ci = rb_vm_ci_lookup(mid, flag, argc, kwarg);
+
+    if (debug) rp(ci);
+    if (kwarg) {
+        RB_DEBUG_COUNTER_INC(ci_kw);
+    }
+    else {
+        RB_DEBUG_COUNTER_INC(ci_nokw);
+    }
+
+    VM_ASSERT(vm_ci_flag(ci) == flag);
+    VM_ASSERT(vm_ci_argc(ci) == argc);
+
+    return ci;
+}
+
+
+static inline const struct rb_callinfo *
+vm_ci_new_runtime_(ID mid, unsigned int flag, unsigned int argc, const struct rb_callinfo_kwarg *kwarg, const char *file, int line)
+{
+    RB_DEBUG_COUNTER_INC(ci_runtime);
+    return vm_ci_new_(mid, flag, argc, kwarg, file, line);
+}
+
+#define VM_CALLINFO_NOT_UNDER_GC IMEMO_FL_USER0
+
+static inline bool
+vm_ci_markable(const struct rb_callinfo *ci)
+{
+    if (! ci) {
+        return false; /* or true? This is Qfalse... */
+    }
+    else if (vm_ci_packed_p(ci)) {
+        return true;
+    }
+    else {
+        VM_ASSERT(IMEMO_TYPE_P(ci, imemo_callinfo));
+        return ! FL_ANY_RAW((VALUE)ci, VM_CALLINFO_NOT_UNDER_GC);
+    }
+}
+
+#define VM_CI_ON_STACK(mid_, flags_, argc_, kwarg_) \
+    (struct rb_callinfo) {                          \
+        .flags = T_IMEMO |                          \
+            (imemo_callinfo << FL_USHIFT) |         \
+            VM_CALLINFO_NOT_UNDER_GC,               \
+        .mid   = mid_,                              \
+        .flag  = flags_,                            \
+        .argc  = argc_,                             \
+        .kwarg = kwarg_,                            \
+    }
+
+typedef VALUE (*vm_call_handler)(
+    struct rb_execution_context_struct *ec,
+    struct rb_control_frame_struct *cfp,
+    struct rb_calling_info *calling);
+
+// imemo_callcache
+
+struct rb_callcache {
+    const VALUE flags;
+
+    /* inline cache: key */
+    const VALUE klass; // Weak reference. When klass is collected, `cc->klass = Qundef`.
+
+    /* inline cache: values */
+    const struct rb_callable_method_entry_struct * const cme_;
+    const vm_call_handler call_;
+
+    union {
+        struct {
+            uint64_t value; // Shape ID in former half, index in latter half
+        } attr;
+        const enum method_missing_reason method_missing_reason; /* used by method_missing */
+        VALUE v;
+        const struct rb_builtin_function *bf;
+    } aux_;
+};
+
+/* VM_CALLCACHE_IVAR used for IVAR/ATTRSET/STRUCT_AREF/STRUCT_ASET methods */
+#define VM_CALLCACHE_IVAR       IMEMO_FL_USER0
+#define VM_CALLCACHE_BF         IMEMO_FL_USER1
+#define VM_CALLCACHE_SUPER      IMEMO_FL_USER2
+#define VM_CALLCACHE_REFINEMENT IMEMO_FL_USER3
+#define VM_CALLCACHE_UNMARKABLE IMEMO_FL_USER4
+#define VM_CALLCACHE_ON_STACK   IMEMO_FL_USER5
+#define VM_CALLCACHE_INVALID_SUPER IMEMO_FL_USER6
+
+enum vm_cc_type {
+    cc_type_normal, // chained from ccs
+    cc_type_super,
+    cc_type_refinement,
+};
+
+extern const struct rb_callcache *rb_vm_empty_cc(void);
+extern const struct rb_callcache *rb_vm_empty_cc_for_super(void);
+
+#define vm_cc_empty() rb_vm_empty_cc()
+
+static inline VALUE
+cc_check_class(VALUE klass)
+{
+    VM_ASSERT(klass == Qundef || RB_TYPE_P(klass, T_CLASS) || RB_TYPE_P(klass, T_ICLASS));
+    return klass;
+}
+
+VALUE rb_vm_cc_table_create(size_t capa);
+VALUE rb_vm_cc_table_dup(VALUE old_table);
+void rb_vm_cc_table_delete(VALUE table, ID mid);
+
+static inline void vm_cc_attr_index_set(const struct rb_callcache *cc, uint64_t packed_cache);
+
+static inline const struct rb_callcache *
+vm_cc_new(VALUE klass,
+          const struct rb_callable_method_entry_struct *cme,
+          vm_call_handler call,
+          enum vm_cc_type type)
+{
+    cc_check_class(klass);
+    struct rb_callcache *cc = SHAREABLE_IMEMO_NEW(struct rb_callcache, imemo_callcache, klass);
+    rb_gc_declare_weak_references((VALUE)cc);
+
+    *((struct rb_callable_method_entry_struct **)&cc->cme_) = (struct rb_callable_method_entry_struct *)cme;
+    *((vm_call_handler *)&cc->call_) = call;
+
+    switch (type) {
+      case cc_type_normal:
+        break;
+      case cc_type_super:
+        *(VALUE *)&cc->flags |= VM_CALLCACHE_SUPER;
+        break;
+      case cc_type_refinement:
+        *(VALUE *)&cc->flags |= VM_CALLCACHE_REFINEMENT;
+        rb_vm_insert_cc_refinement(cc);
+        break;
+    }
+
+    if (cme) {
+        if (cme->def->type == VM_METHOD_TYPE_ATTRSET) {
+            vm_cc_attr_index_set(cc, IVAR_CACHE_INIT);
+        }
+        else if (cme->def->type == VM_METHOD_TYPE_IVAR) {
+            vm_cc_attr_index_set(cc, rb_getivar_cache_pack(ROOT_SHAPE_ID, ATTR_INDEX_NOT_SET));
+        }
+    }
+    else {
+        *(VALUE *)&cc->flags |= VM_CALLCACHE_INVALID_SUPER;
+    }
+
+    RB_DEBUG_COUNTER_INC(cc_new);
+    return cc;
+}
+
+static inline bool
+vm_cc_super_p(const struct rb_callcache *cc)
+{
+    return (cc->flags & VM_CALLCACHE_SUPER) != 0;
+}
+
+static inline bool
+vm_cc_refinement_p(const struct rb_callcache *cc)
+{
+    return (cc->flags & VM_CALLCACHE_REFINEMENT) != 0;
+}
+
+#define VM_CC_ON_STACK(clazz, call, aux, cme) \
+    (struct rb_callcache) {                   \
+        .flags = T_IMEMO |                    \
+            (imemo_callcache << FL_USHIFT) |  \
+            VM_CALLCACHE_UNMARKABLE |         \
+            VM_CALLCACHE_ON_STACK,            \
+        .klass = cc_check_class(clazz),       \
+        .cme_  = cme,                         \
+        .call_ = call,                        \
+        .aux_  = aux,                         \
+    }
+
+static inline bool
+vm_cc_class_check(const struct rb_callcache *cc, VALUE klass)
+{
+    VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
+    VM_ASSERT(cc_check_class(cc->klass));
+    return cc->klass == klass;
+}
+
+static inline int
+vm_cc_markable(const struct rb_callcache *cc)
+{
+    VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
+    return FL_TEST_RAW((VALUE)cc, VM_CALLCACHE_UNMARKABLE) == 0;
+}
+
+static inline bool
+vm_cc_invalid_super(const struct rb_callcache *cc)
+{
+    VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
+    // Set when calling super and there is no superclass.
+    return FL_TEST_RAW((VALUE)cc, VM_CALLCACHE_INVALID_SUPER);
+}
+
+static inline bool
+vm_cc_valid(const struct rb_callcache *cc)
+{
+    VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
+    VM_ASSERT(cc_check_class(cc->klass));
+
+    return !UNDEF_P(cc->klass);
+}
+
+/* Whether another ractor may have invalidated this cc mid-use (see
+ * vm_cc_invalidate()). Extensions cannot use rb_multi_ractor_p() here: with
+ * assertions enabled at -O0 (e.g. --enable-yjit=dev on macOS) the inline
+ * materializes references to core-internal globals (ruby_current_vm_ptr,
+ * ruby_single_main_ractor) that are not exported to them, and objspace.bundle
+ * fails to load. RUBY_EXPORT marks a core TU. */
+#ifdef RUBY_EXPORT
+# define VM_CC_RACED_P() rb_multi_ractor_p()
+#else
+# define VM_CC_RACED_P() true
+#endif
+
+static inline const struct rb_callable_method_entry_struct *
+vm_cc_cme(const struct rb_callcache *cc)
+{
+    VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
+    // VM_CC_RACED_P(): another ractor can invalidate this cc (klass = Qundef)
+    // while a call using it is in flight; cme_/call_ stay intact, so the call
+    // proceeds with the method resolved before the redefinition.
+    VM_ASSERT(cc->klass != Qundef || !vm_cc_markable(cc) || vm_cc_invalid_super(cc) || VM_CC_RACED_P());
+    VM_ASSERT(cc_check_class(cc->klass));
+    VM_ASSERT(cc->call_ == NULL   || // not initialized yet
+              !vm_cc_markable(cc) ||
+              vm_cc_invalid_super(cc) ||
+              cc->cme_ != NULL);
+
+    return cc->cme_;
+}
+
+static inline vm_call_handler
+vm_cc_call(const struct rb_callcache *cc)
+{
+    VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
+    VM_ASSERT(cc->call_ != NULL);
+    VM_ASSERT(cc->klass != Qundef || !vm_cc_markable(cc) || vm_cc_invalid_super(cc) || VM_CC_RACED_P());
+    VM_ASSERT(cc_check_class(cc->klass));
+    return cc->call_;
+}
+
+static inline uint64_t
+vm_cc_atomic_cache_read(const struct rb_callcache *cc)
+{
+    return ATOMIC_U64_LOAD_RELAXED(cc->aux_.attr.value);
+}
+
+static inline uint64_t
+vm_ic_atomic_cache_read(const struct iseq_inline_iv_cache_entry *ic)
+{
+    return ATOMIC_U64_LOAD_RELAXED(ic->value);
+}
+
+static inline uint64_t
+vm_cache_attr_index_atomic_read(bool is_attr, const struct iseq_inline_iv_cache_entry *ic, const struct rb_callcache *cc)
+{
+    if (is_attr) {
+        return vm_cc_atomic_cache_read(cc);
+    }
+    else {
+        return vm_ic_atomic_cache_read(ic);
+    }
+}
+
+static inline unsigned int
+vm_cc_cmethod_missing_reason(const struct rb_callcache *cc)
+{
+    VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
+    return cc->aux_.method_missing_reason;
+}
+
+static inline bool
+vm_cc_invalidated_p(const struct rb_callcache *cc)
+{
+    if (vm_cc_valid(cc) && !METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc))) {
+        return false;
+    }
+    else {
+        return true;
+    }
+}
+
+/* callcache: mutate */
+
+static inline void
+vm_cc_call_set(const struct rb_callcache *cc, vm_call_handler call)
+{
+    VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
+    VM_ASSERT(cc != vm_cc_empty());
+    *(vm_call_handler *)&cc->call_ = call;
+}
+
+static inline void
+set_vm_cc_ivar(const struct rb_callcache *cc)
+{
+    *(VALUE *)&cc->flags |= VM_CALLCACHE_IVAR;
+}
+
+static inline bool
+vm_cc_ivar_p(const struct rb_callcache *cc)
+{
+    return (cc->flags & VM_CALLCACHE_IVAR) != 0;
+}
+
+static inline void
+vm_cc_attr_index_set(const struct rb_callcache *cc, uint64_t packed_cache)
+{
+    uint64_t *attr_value = (uint64_t *)&cc->aux_.attr.value;
+    if (!vm_cc_markable(cc)) {
+        *attr_value = IVAR_CACHE_INIT;
+        return;
+    }
+    VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
+    VM_ASSERT(cc != vm_cc_empty());
+    *attr_value = packed_cache;
+    set_vm_cc_ivar(cc);
+}
+
+static inline void
+vm_cache_attr_index_set(bool is_attr, struct iseq_inline_iv_cache_entry *ic, const struct rb_callcache *cc, uint64_t packed_cache)
+{
+    if (is_attr) {
+        vm_cc_attr_index_set(cc, packed_cache);
+    }
+    else {
+        ATOMIC_U64_SET_RELAXED(ic->value, packed_cache);
+    }
+}
+
+static inline void
+vm_cc_method_missing_reason_set(const struct rb_callcache *cc, enum method_missing_reason reason)
+{
+    VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
+    VM_ASSERT(cc != vm_cc_empty());
+    *(enum method_missing_reason *)&cc->aux_.method_missing_reason = reason;
+}
+
+static inline void
+vm_cc_bf_set(const struct rb_callcache *cc, const struct rb_builtin_function *bf)
+{
+    VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
+    VM_ASSERT(cc != vm_cc_empty());
+    *(const struct rb_builtin_function **)&cc->aux_.bf = bf;
+    *(VALUE *)&cc->flags |= VM_CALLCACHE_BF;
+}
+
+static inline bool
+vm_cc_bf_p(const struct rb_callcache *cc)
+{
+    return (cc->flags & VM_CALLCACHE_BF) != 0;
+}
+
+static inline void
+vm_cc_invalidate(const struct rb_callcache *cc)
+{
+    VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
+    VM_ASSERT(cc != vm_cc_empty());
+    // TODO: rb_multi_ractor_p() is a workaround to stabilize CI
+    VM_ASSERT(cc->klass != Qundef || rb_multi_ractor_p()); // should be enable
+
+    *(VALUE *)&cc->klass = Qundef;
+    RB_DEBUG_COUNTER_INC(cc_ent_invalidate);
+}
+
+/* calldata */
+
+struct rb_call_data {
+    const struct rb_callinfo *ci;
+    const struct rb_callcache *cc;
+};
+
+struct rb_class_cc_entries {
+#if VM_CHECK_MODE > 0
+    VALUE debug_sig;
+#endif
+    int capa;
+    int len;
+    const struct rb_callable_method_entry_struct *cme;
+    struct rb_class_cc_entries_entry {
+        const struct rb_callcache *cc;
+        unsigned int argc;
+        unsigned short flag;
+        unsigned short kw_len;
+    } entries[FLEX_ARY_LEN];
+};
+
+/* entries[].flag is an unsigned short, so every VM_CALL flag bit must fit in 16 bits. */
+STATIC_ASSERT(cc_entries_flag_fits_in_short, VM_CALL__END <= 16);
+
+/* entries[].kw_len is an unsigned short, so a call site cannot carry, nor a method
+   declare, more keyword arguments than this.  Enforced at compile time. */
+#define VM_CALL_KW_LEN_MAX UINT16_MAX
+
+static inline size_t
+vm_ccs_alloc_size(size_t capa)
+{
+    return offsetof(struct rb_class_cc_entries, entries) + (sizeof(struct rb_class_cc_entries_entry) * capa);
+}
+
+#if VM_CHECK_MODE > 0
+
+const rb_callable_method_entry_t *rb_vm_lookup_overloaded_cme(const rb_callable_method_entry_t *cme);
+void rb_vm_dump_overloaded_cme_table(void);
+
+static inline bool
+vm_ccs_p(const struct rb_class_cc_entries *ccs)
+{
+    return ccs->debug_sig == ~(VALUE)ccs;
+}
+
+static inline bool
+vm_cc_check_cme_unlocked(const struct rb_callcache *cc, const rb_callable_method_entry_t *cme)
+{
+    return vm_cc_cme(cc) == cme ||
+        (cme->def->iseq_overload && vm_cc_cme(cc) == rb_vm_lookup_overloaded_cme(cme));
+}
+
+static inline bool
+vm_cc_check_cme(const struct rb_callcache *cc, const rb_callable_method_entry_t *cme)
+{
+    bool valid;
+    if (rb_current_execution_context(false) == NULL) {
+        valid = vm_cc_check_cme_unlocked(cc, cme);
+    }
+    else {
+        RB_VM_LOCKING_NO_BARRIER() {
+            valid = vm_cc_check_cme_unlocked(cc, cme);
+        }
+    }
+    if (valid) {
+        return true;
+    }
+#if 1
+        // debug print
+
+        fprintf(stderr, "iseq_overload:%d, cme:%p (def:%p), cm_cc_cme(cc):%p (def:%p)\n",
+                (int)cme->def->iseq_overload,
+                (void *)cme, (void *)cme->def,
+                (void *)vm_cc_cme(cc), (void *)vm_cc_cme(cc)->def);
+        rp(cme);
+        rp(vm_cc_cme(cc));
+        rp(rb_vm_lookup_overloaded_cme(cme));
+#endif
+    return false;
+}
+
+#endif
+
+#endif /* RUBY_VM_CALLINFO_H */