llama_cpp 0.1.4 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,10 +1,19 @@
1
+ #pragma once
2
+
1
3
  #include "ggml.h"
2
4
 
3
5
  #ifdef __cplusplus
4
6
  extern "C" {
5
7
  #endif
6
8
 
9
+ #define GGML_CUDA_MAX_DEVICES 16
10
+
11
+ struct ggml_tensor_extra_gpu {
12
+ void * data_device[GGML_CUDA_MAX_DEVICES]; // 1 pointer for each device for split tensors
13
+ };
14
+
7
15
  void ggml_init_cublas(void);
16
+ void ggml_cuda_set_tensor_split(const float * tensor_split);
8
17
 
9
18
  void ggml_cuda_mul(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);
10
19
  bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);
@@ -15,8 +24,15 @@ void ggml_cuda_mul_mat(const struct ggml_tensor * src0, const struct ggml_tens
15
24
  void * ggml_cuda_host_malloc(size_t size);
16
25
  void ggml_cuda_host_free(void * ptr);
17
26
 
18
- void ggml_cuda_transform_tensor(struct ggml_tensor * tensor);
19
- void ggml_cuda_load_data(const char * fname, struct ggml_tensor * tensors, size_t offset);
27
+ void ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor);
28
+
29
+ void ggml_cuda_free_data(struct ggml_tensor * tensor);
30
+ void ggml_cuda_assign_buffers(struct ggml_tensor * tensor);
31
+ void ggml_cuda_assign_buffers_no_scratch(struct ggml_tensor * tensor);
32
+ void ggml_cuda_set_main_device(int main_device);
33
+ void ggml_cuda_set_scratch_size(size_t scratch_size);
34
+ void ggml_cuda_free_scratch(void);
35
+ bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor);
20
36
 
21
37
  #ifdef __cplusplus
22
38
  }
@@ -0,0 +1,64 @@
1
+ // An interface allowing to compute ggml_cgraph with Metal
2
+ //
3
+ // This is a fully functional interface that extends ggml with GPU support for Apple devices.
4
+ // A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, OpenCL, etc.)
5
+ //
6
+ // How it works?
7
+ //
8
+ // As long as your program can create and evaluate a ggml_cgraph on the CPU, you can use this
9
+ // interface to evaluate the same graph on the GPU. Instead of using ggml_graph_compute(), you
10
+ // use ggml_metal_graph_compute() (or ggml_vulkan_graph_compute(), etc.)
11
+ //
12
+ // You only need to make sure that all memory buffers that you used during the graph creation
13
+ // are mapped to the device memory with the ggml_metal_add_buffer() function. This mapping is
14
+ // used during the graph evaluation to determine the arguments of the compute kernels.
15
+ //
16
+ // Synchronization between device and host memory (for example for input and output tensors)
17
+ // is done with the ggml_metal_set_tensor() and ggml_metal_get_tensor() functions.
18
+ //
19
+
20
+ #pragma once
21
+
22
+ #include <stddef.h>
23
+ #include <stdbool.h>
24
+
25
+ // max memory buffers that can be mapped to the device
26
+ #define GGML_METAL_MAX_BUFFERS 16
27
+
28
+ struct ggml_tensor;
29
+ struct ggml_cgraph;
30
+
31
+ #ifdef __cplusplus
32
+ extern "C" {
33
+ #endif
34
+
35
+ struct ggml_metal_context;
36
+
37
+ struct ggml_metal_context * ggml_metal_init(void);
38
+ void ggml_metal_free(struct ggml_metal_context * ctx);
39
+
40
+ // creates a mapping between a host memory buffer and a device memory buffer
41
+ // - make sure to map all buffers used in the graph before calling ggml_metal_graph_compute
42
+ // - the mapping is used during computation to determine the arguments of the compute kernels
43
+ // - you don't need to keep the host memory buffer allocated as it is never accessed by Metal
44
+ //
45
+ bool ggml_metal_add_buffer(
46
+ struct ggml_metal_context * ctx,
47
+ const char * name,
48
+ void * data,
49
+ size_t size);
50
+
51
+ // set data from host memory into the device
52
+ void ggml_metal_set_tensor(struct ggml_metal_context * ctx, struct ggml_tensor * t);
53
+
54
+ // get data from the device into host memory
55
+ void ggml_metal_get_tensor(struct ggml_metal_context * ctx, struct ggml_tensor * t);
56
+
57
+ // same as ggml_graph_compute but uses Metal
58
+ // creates gf->n_threads command buffers in parallel
59
+ void ggml_metal_graph_compute(struct ggml_metal_context * ctx, struct ggml_cgraph * gf);
60
+
61
+ #ifdef __cplusplus
62
+ }
63
+ #endif
64
+